You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by bi...@apache.org on 2014/08/16 02:54:49 UTC
[01/10] TEZ-1055. Rename tez-mapreduce-examples to tez-examples
(Hitesh Shah via bikas)
Repository: tez
Updated Branches:
refs/heads/master 4a41ccf06 -> 41f5cd8a2
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraGen.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraGen.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraGen.java
new file mode 100644
index 0000000..70a4207
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraGen.java
@@ -0,0 +1,311 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples.terasort;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.zip.Checksum;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.PureJavaCrc32;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * Generate the official GraySort input data set.
+ * The user specifies the number of rows and the output directory and this
+ * class runs a map/reduce program to generate the data.
+ * The format of the data is:
+ * <ul>
+ * <li>(10 bytes key) (constant 2 bytes) (32 bytes rowid)
+ * (constant 4 bytes) (48 bytes filler) (constant 4 bytes)
+ * <li>The rowid is the right justified row id as a hex number.
+ * </ul>
+ *
+ * <p>
+ * To run the program:
+ * <b>bin/hadoop jar hadoop-*-examples.jar teragen 10000000000 in-dir</b>
+ */
+public class TeraGen extends Configured implements Tool {
+ private static final Log LOG = LogFactory.getLog(TeraSort.class);
+
+ public static enum Counters {CHECKSUM}
+
+ public static final String NUM_ROWS = "mapreduce.terasort.num-rows";
+ /**
+ * An input format that assigns ranges of longs to each mapper.
+ */
+ static class RangeInputFormat
+ extends InputFormat<LongWritable, NullWritable> {
+
+ /**
+ * An input split consisting of a range on numbers.
+ */
+ static class RangeInputSplit extends InputSplit implements Writable {
+ long firstRow;
+ long rowCount;
+
+ public RangeInputSplit() { }
+
+ public RangeInputSplit(long offset, long length) {
+ firstRow = offset;
+ rowCount = length;
+ }
+
+ public long getLength() throws IOException {
+ return 0;
+ }
+
+ public String[] getLocations() throws IOException {
+ return new String[]{};
+ }
+
+ public void readFields(DataInput in) throws IOException {
+ firstRow = WritableUtils.readVLong(in);
+ rowCount = WritableUtils.readVLong(in);
+ }
+
+ public void write(DataOutput out) throws IOException {
+ WritableUtils.writeVLong(out, firstRow);
+ WritableUtils.writeVLong(out, rowCount);
+ }
+ }
+
+ /**
+ * A record reader that will generate a range of numbers.
+ */
+ static class RangeRecordReader
+ extends RecordReader<LongWritable, NullWritable> {
+ long startRow;
+ long finishedRows;
+ long totalRows;
+ LongWritable key = null;
+
+ public RangeRecordReader() {
+ }
+
+ public void initialize(InputSplit split, TaskAttemptContext context)
+ throws IOException, InterruptedException {
+ startRow = ((RangeInputSplit)split).firstRow;
+ finishedRows = 0;
+ totalRows = ((RangeInputSplit)split).rowCount;
+ }
+
+ public void close() throws IOException {
+ // NOTHING
+ }
+
+ public LongWritable getCurrentKey() {
+ return key;
+ }
+
+ public NullWritable getCurrentValue() {
+ return NullWritable.get();
+ }
+
+ public float getProgress() throws IOException {
+ return finishedRows / (float) totalRows;
+ }
+
+ public boolean nextKeyValue() {
+ if (key == null) {
+ key = new LongWritable();
+ }
+ if (finishedRows < totalRows) {
+ key.set(startRow + finishedRows);
+ finishedRows += 1;
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ }
+
+ public RecordReader<LongWritable, NullWritable>
+ createRecordReader(InputSplit split, TaskAttemptContext context)
+ throws IOException {
+ return new RangeRecordReader();
+ }
+
+ /**
+ * Create the desired number of splits, dividing the number of rows
+ * between the mappers.
+ */
+ public List<InputSplit> getSplits(JobContext job) {
+ long totalRows = getNumberOfRows(job);
+ int numSplits = job.getConfiguration().getInt(MRJobConfig.NUM_MAPS, 1);
+ LOG.info("Generating " + totalRows + " using " + numSplits);
+ List<InputSplit> splits = new ArrayList<InputSplit>();
+ long currentRow = 0;
+ for(int split = 0; split < numSplits; ++split) {
+ long goal =
+ (long) Math.ceil(totalRows * (double)(split + 1) / numSplits);
+ splits.add(new RangeInputSplit(currentRow, goal - currentRow));
+ currentRow = goal;
+ }
+ return splits;
+ }
+
+ }
+
+ static long getNumberOfRows(JobContext job) {
+ return job.getConfiguration().getLong(NUM_ROWS, 0);
+ }
+
+ static void setNumberOfRows(Job job, long numRows) {
+ job.getConfiguration().setLong(NUM_ROWS, numRows);
+ }
+
+ /**
+ * The Mapper class that given a row number, will generate the appropriate
+ * output line.
+ */
+ public static class SortGenMapper
+ extends Mapper<LongWritable, NullWritable, Text, Text> {
+
+ private Text key = new Text();
+ private Text value = new Text();
+ private Unsigned16 rand = null;
+ private Unsigned16 rowId = null;
+ private Unsigned16 checksum = new Unsigned16();
+ private Checksum crc32 = new PureJavaCrc32();
+ private Unsigned16 total = new Unsigned16();
+ private static final Unsigned16 ONE = new Unsigned16(1);
+ private byte[] buffer = new byte[TeraInputFormat.KEY_LENGTH +
+ TeraInputFormat.VALUE_LENGTH];
+ private Counter checksumCounter;
+
+ public void map(LongWritable row, NullWritable ignored,
+ Context context) throws IOException, InterruptedException {
+ if (rand == null) {
+ rowId = new Unsigned16(row.get());
+ rand = Random16.skipAhead(rowId);
+ checksumCounter = context.getCounter(Counters.CHECKSUM);
+ }
+ Random16.nextRand(rand);
+ GenSort.generateRecord(buffer, rand, rowId);
+ key.set(buffer, 0, TeraInputFormat.KEY_LENGTH);
+ value.set(buffer, TeraInputFormat.KEY_LENGTH,
+ TeraInputFormat.VALUE_LENGTH);
+ context.write(key, value);
+ crc32.reset();
+ crc32.update(buffer, 0,
+ TeraInputFormat.KEY_LENGTH + TeraInputFormat.VALUE_LENGTH);
+ checksum.set(crc32.getValue());
+ total.add(checksum);
+ rowId.add(ONE);
+ }
+
+ @Override
+ public void cleanup(Context context) {
+ if (checksumCounter != null) {
+ checksumCounter.increment(total.getLow8());
+ }
+ }
+ }
+
+ private static void usage() throws IOException {
+ System.err.println("teragen <num rows> <output dir>");
+ }
+
+ /**
+ * Parse a number that optionally has a postfix that denotes a base.
+ * @param str an string integer with an option base {k,m,b,t}.
+ * @return the expanded value
+ */
+ private static long parseHumanLong(String str) {
+ char tail = str.charAt(str.length() - 1);
+ long base = 1;
+ switch (tail) {
+ case 't':
+ base *= 1000 * 1000 * 1000 * 1000;
+ break;
+ case 'b':
+ base *= 1000 * 1000 * 1000;
+ break;
+ case 'm':
+ base *= 1000 * 1000;
+ break;
+ case 'k':
+ base *= 1000;
+ break;
+ default:
+ }
+ if (base != 1) {
+ str = str.substring(0, str.length() - 1);
+ }
+ return Long.parseLong(str) * base;
+ }
+
+ /**
+ * @param args the cli arguments
+ */
+ public int run(String[] args)
+ throws IOException, InterruptedException, ClassNotFoundException {
+ Job job = Job.getInstance(getConf());
+ if (args.length != 2) {
+ usage();
+ return 2;
+ }
+ setNumberOfRows(job, parseHumanLong(args[0]));
+ Path outputDir = new Path(args[1]);
+ if (outputDir.getFileSystem(getConf()).exists(outputDir)) {
+ throw new IOException("Output directory " + outputDir +
+ " already exists.");
+ }
+ FileOutputFormat.setOutputPath(job, outputDir);
+ job.setJobName("TeraGen");
+ job.setJarByClass(TeraGen.class);
+ job.setMapperClass(SortGenMapper.class);
+ job.setNumReduceTasks(0);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(Text.class);
+ job.setInputFormatClass(RangeInputFormat.class);
+ job.setOutputFormatClass(TeraOutputFormat.class);
+ return job.waitForCompletion(true) ? 0 : 1;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(), new TeraGen(), args);
+ System.exit(res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraInputFormat.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraInputFormat.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraInputFormat.java
new file mode 100644
index 0000000..94ee340
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraInputFormat.java
@@ -0,0 +1,353 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples.terasort;
+
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+import org.apache.hadoop.util.IndexedSortable;
+import org.apache.hadoop.util.QuickSort;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * An input format that reads the first 10 characters of each line as the key
+ * and the rest of the line as the value. Both key and value are represented
+ * as Text.
+ */
+public class TeraInputFormat extends FileInputFormat<Text,Text> {
+
+ static final String PARTITION_FILENAME = "_partition.lst";
+ private static final String NUM_PARTITIONS =
+ "mapreduce.terasort.num.partitions";
+ private static final String SAMPLE_SIZE =
+ "mapreduce.terasort.partitions.sample";
+ static final int KEY_LENGTH = 10;
+ static final int VALUE_LENGTH = 90;
+ static final int RECORD_LENGTH = KEY_LENGTH + VALUE_LENGTH;
+ private static MRJobConfig lastContext = null;
+ private static List<InputSplit> lastResult = null;
+
+ static class TeraFileSplit extends FileSplit {
+ static private String[] ZERO_LOCATIONS = new String[0];
+
+ private String[] locations;
+
+ public TeraFileSplit() {
+ locations = ZERO_LOCATIONS;
+ }
+ public TeraFileSplit(Path file, long start, long length, String[] hosts) {
+ super(file, start, length, hosts);
+ try {
+ locations = super.getLocations();
+ } catch (IOException e) {
+ locations = ZERO_LOCATIONS;
+ }
+ }
+
+ // XXXXXX should this also be null-protected?
+ protected void setLocations(String[] hosts) {
+ locations = hosts;
+ }
+
+ @Override
+ public String[] getLocations() {
+ return locations;
+ }
+
+ public String toString() {
+ StringBuffer result = new StringBuffer();
+ result.append(getPath());
+ result.append(" from ");
+ result.append(getStart());
+ result.append(" length ");
+ result.append(getLength());
+ for(String host: getLocations()) {
+ result.append(" ");
+ result.append(host);
+ }
+ return result.toString();
+ }
+ }
+
+ static class TextSampler implements IndexedSortable {
+ private ArrayList<Text> records = new ArrayList<Text>();
+
+ public int compare(int i, int j) {
+ Text left = records.get(i);
+ Text right = records.get(j);
+ return left.compareTo(right);
+ }
+
+ public void swap(int i, int j) {
+ Text left = records.get(i);
+ Text right = records.get(j);
+ records.set(j, left);
+ records.set(i, right);
+ }
+
+ public void addKey(Text key) {
+ synchronized (this) {
+ records.add(new Text(key));
+ }
+ }
+
+ /**
+ * Find the split points for a given sample. The sample keys are sorted
+ * and down sampled to find even split points for the partitions. The
+ * returned keys should be the start of their respective partitions.
+ * @param numPartitions the desired number of partitions
+ * @return an array of size numPartitions - 1 that holds the split points
+ */
+ Text[] createPartitions(int numPartitions) {
+ int numRecords = records.size();
+ System.out.println("Making " + numPartitions + " from " + numRecords +
+ " sampled records");
+ if (numPartitions > numRecords) {
+ throw new IllegalArgumentException
+ ("Requested more partitions than input keys (" + numPartitions +
+ " > " + numRecords + ")");
+ }
+ new QuickSort().sort(this, 0, records.size());
+ float stepSize = numRecords / (float) numPartitions;
+ Text[] result = new Text[numPartitions-1];
+ for(int i=1; i < numPartitions; ++i) {
+ result[i-1] = records.get(Math.round(stepSize * i));
+ }
+ return result;
+ }
+ }
+
+ /**
+ * Use the input splits to take samples of the input and generate sample
+ * keys. By default reads 100,000 keys from 10 locations in the input, sorts
+ * them and picks N-1 keys to generate N equally sized partitions.
+ * @param job the job to sample
+ * @param partFile where to write the output file to
+ * @throws Throwable if something goes wrong
+ */
+ public static void writePartitionFile(final JobContext job,
+ Path partFile) throws Throwable {
+ long t1 = System.currentTimeMillis();
+ Configuration conf = job.getConfiguration();
+ final TeraInputFormat inFormat = new TeraInputFormat();
+ final TextSampler sampler = new TextSampler();
+ int partitions = job.getNumReduceTasks();
+ long sampleSize = conf.getLong(SAMPLE_SIZE, 100000);
+ final List<InputSplit> splits = inFormat.getSplits(job);
+ long t2 = System.currentTimeMillis();
+ System.out.println("Computing input splits took " + (t2 - t1) + "ms");
+ int samples = Math.min(conf.getInt(NUM_PARTITIONS, 10), splits.size());
+ System.out.println("Sampling " + samples + " splits of " + splits.size());
+ final long recordsPerSample = sampleSize / samples;
+ final int sampleStep = splits.size() / samples;
+ Thread[] samplerReader = new Thread[samples];
+ SamplerThreadGroup threadGroup = new SamplerThreadGroup("Sampler Reader Thread Group");
+ // take N samples from different parts of the input
+ for(int i=0; i < samples; ++i) {
+ final int idx = i;
+ samplerReader[i] =
+ new Thread (threadGroup,"Sampler Reader " + idx) {
+ {
+ setDaemon(true);
+ }
+ public void run() {
+ long records = 0;
+ try {
+ TaskAttemptContext context = new TaskAttemptContextImpl(
+ job.getConfiguration(), new TaskAttemptID());
+ RecordReader<Text, Text> reader =
+ inFormat.createRecordReader(splits.get(sampleStep * idx),
+ context);
+ reader.initialize(splits.get(sampleStep * idx), context);
+ while (reader.nextKeyValue()) {
+ sampler.addKey(new Text(reader.getCurrentKey()));
+ records += 1;
+ if (recordsPerSample <= records) {
+ break;
+ }
+ }
+ } catch (IOException ie){
+ System.err.println("Got an exception while reading splits " +
+ StringUtils.stringifyException(ie));
+ throw new RuntimeException(ie);
+ } catch (InterruptedException e) {
+
+ }
+ }
+ };
+ samplerReader[i].start();
+ }
+ FileSystem outFs = partFile.getFileSystem(conf);
+ DataOutputStream writer = outFs.create(partFile, true, 64*1024, (short) 10,
+ outFs.getDefaultBlockSize(partFile));
+ for (int i = 0; i < samples; i++) {
+ try {
+ samplerReader[i].join();
+ if(threadGroup.getThrowable() != null){
+ throw threadGroup.getThrowable();
+ }
+ } catch (InterruptedException e) {
+ }
+ }
+ for(Text split : sampler.createPartitions(partitions)) {
+ split.write(writer);
+ }
+ writer.close();
+ long t3 = System.currentTimeMillis();
+ System.out.println("Computing parititions took " + (t3 - t2) + "ms");
+ }
+
+ static class SamplerThreadGroup extends ThreadGroup{
+
+ private Throwable throwable;
+
+ public SamplerThreadGroup(String s) {
+ super(s);
+ }
+
+ @Override
+ public void uncaughtException(Thread thread, Throwable throwable) {
+ this.throwable = throwable;
+ }
+
+ public Throwable getThrowable() {
+ return this.throwable;
+ }
+
+ }
+
+ static class TeraRecordReader extends RecordReader<Text,Text> {
+ private FSDataInputStream in;
+ private long offset;
+ private long length;
+ private static final int RECORD_LENGTH = KEY_LENGTH + VALUE_LENGTH;
+ private byte[] buffer = new byte[RECORD_LENGTH];
+ private Text key;
+ private Text value;
+
+ public TeraRecordReader() throws IOException {
+ }
+
+ public void initialize(InputSplit split, TaskAttemptContext context)
+ throws IOException, InterruptedException {
+ Path p = ((FileSplit)split).getPath();
+ FileSystem fs = p.getFileSystem(context.getConfiguration());
+ in = fs.open(p);
+ long start = ((FileSplit)split).getStart();
+ // find the offset to start at a record boundary
+ offset = (RECORD_LENGTH - (start % RECORD_LENGTH)) % RECORD_LENGTH;
+ in.seek(start + offset);
+ length = ((FileSplit)split).getLength();
+ }
+
+ public void close() throws IOException {
+ in.close();
+ }
+
+ public Text getCurrentKey() {
+ return key;
+ }
+
+ public Text getCurrentValue() {
+ return value;
+ }
+
+ public float getProgress() throws IOException {
+ return (float) offset / length;
+ }
+
+ public boolean nextKeyValue() throws IOException {
+ if (offset >= length) {
+ return false;
+ }
+ int read = 0;
+ while (read < RECORD_LENGTH) {
+ long newRead = in.read(buffer, read, RECORD_LENGTH - read);
+ if (newRead == -1) {
+ if (read == 0) {
+ return false;
+ } else {
+ throw new EOFException("read past eof");
+ }
+ }
+ read += newRead;
+ }
+ if (key == null) {
+ key = new Text();
+ }
+ if (value == null) {
+ value = new Text();
+ }
+ key.set(buffer, 0, KEY_LENGTH);
+ value.set(buffer, KEY_LENGTH, VALUE_LENGTH);
+ offset += RECORD_LENGTH;
+ return true;
+ }
+ }
+
+ @Override
+ public RecordReader<Text, Text>
+ createRecordReader(InputSplit split, TaskAttemptContext context)
+ throws IOException {
+ return new TeraRecordReader();
+ }
+
+ protected FileSplit makeSplit(Path file, long start, long length,
+ String[] hosts) {
+ return new TeraFileSplit(file, start, length, hosts);
+ }
+
+ @Override
+ public List<InputSplit> getSplits(JobContext job) throws IOException {
+ if (job == lastContext) {
+ return lastResult;
+ }
+ long t1, t2, t3;
+ t1 = System.currentTimeMillis();
+ lastContext = job;
+ lastResult = super.getSplits(job);
+ t2 = System.currentTimeMillis();
+ System.out.println("Spent " + (t2 - t1) + "ms computing base-splits.");
+ if (job.getConfiguration().getBoolean(TeraScheduler.USE, true)) {
+ TeraScheduler scheduler = new TeraScheduler(
+ lastResult.toArray(new TeraFileSplit[0]), job.getConfiguration());
+ lastResult = scheduler.getNewFileSplits();
+ t3 = System.currentTimeMillis();
+ System.out.println("Spent " + (t3 - t2) + "ms computing TeraScheduler splits.");
+ }
+ return lastResult;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraOutputFormat.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraOutputFormat.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraOutputFormat.java
new file mode 100644
index 0000000..4ff0f06
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraOutputFormat.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples.terasort;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.InvalidJobConfException;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.security.TokenCache;
+
+/**
+ * An output format that writes the key and value appended together.
+ */
+public class TeraOutputFormat extends FileOutputFormat<Text,Text> {
+ static final String FINAL_SYNC_ATTRIBUTE = "mapreduce.terasort.final.sync";
+ private OutputCommitter committer = null;
+
+ /**
+ * Set the requirement for a final sync before the stream is closed.
+ */
+ static void setFinalSync(JobContext job, boolean newValue) {
+ job.getConfiguration().setBoolean(FINAL_SYNC_ATTRIBUTE, newValue);
+ }
+
+ /**
+ * Does the user want a final sync at close?
+ */
+ public static boolean getFinalSync(JobContext job) {
+ return job.getConfiguration().getBoolean(FINAL_SYNC_ATTRIBUTE, false);
+ }
+
+ static class TeraRecordWriter extends RecordWriter<Text,Text> {
+ private boolean finalSync = false;
+ private FSDataOutputStream out;
+
+ public TeraRecordWriter(FSDataOutputStream out,
+ JobContext job) {
+ finalSync = getFinalSync(job);
+ this.out = out;
+ }
+
+ public synchronized void write(Text key,
+ Text value) throws IOException {
+ out.write(key.getBytes(), 0, key.getLength());
+ out.write(value.getBytes(), 0, value.getLength());
+ }
+
+ public void close(TaskAttemptContext context) throws IOException {
+ if (finalSync) {
+ out.hsync();
+ }
+ out.close();
+ }
+ }
+
+ @Override
+ public void checkOutputSpecs(JobContext job
+ ) throws InvalidJobConfException, IOException {
+ // Ensure that the output directory is set
+ Path outDir = getOutputPath(job);
+ if (outDir == null) {
+ throw new InvalidJobConfException("Output directory not set in JobConf.");
+ }
+
+ // get delegation token for outDir's file system
+ TokenCache.obtainTokensForNamenodes(job.getCredentials(),
+ new Path[] { outDir }, job.getConfiguration());
+ }
+
+ public RecordWriter<Text,Text> getRecordWriter(TaskAttemptContext job
+ ) throws IOException {
+ Path file = getDefaultWorkFile(job, "");
+ FileSystem fs = file.getFileSystem(job.getConfiguration());
+ FSDataOutputStream fileOut = fs.create(file);
+ return new TeraRecordWriter(fileOut, job);
+ }
+
+ public OutputCommitter getOutputCommitter(TaskAttemptContext context)
+ throws IOException {
+ if (committer == null) {
+ Path output = getOutputPath(context);
+ committer = new FileOutputCommitter(output, context);
+ }
+ return committer;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraScheduler.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraScheduler.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraScheduler.java
new file mode 100644
index 0000000..82638d8
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraScheduler.java
@@ -0,0 +1,253 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples.terasort;
+
+import java.io.*;
+import java.util.*;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
+import org.apache.tez.mapreduce.examples.terasort.TeraInputFormat.TeraFileSplit;
+
+import com.google.common.base.Charsets;
+
+class TeraScheduler {
+ static String USE = "mapreduce.terasort.use.terascheduler";
+ private static final Log LOG = LogFactory.getLog(TeraScheduler.class);
+ private Split[] splits;
+ private List<Host> hosts = new ArrayList<Host>();
+ private int slotsPerHost;
+ private int remainingSplits = 0;
+ private FileSplit[] realSplits = null;
+
+ static class Split {
+ String filename;
+ boolean isAssigned = false;
+ List<Host> locations = new ArrayList<Host>();
+ Split(String filename) {
+ this.filename = filename;
+ }
+ public String toString() {
+ StringBuffer result = new StringBuffer();
+ result.append(filename);
+ result.append(" on ");
+ for(Host host: locations) {
+ result.append(host.hostname);
+ result.append(", ");
+ }
+ return result.toString();
+ }
+ }
+ static class Host {
+ String hostname;
+ List<Split> splits = new ArrayList<Split>();
+ Host(String hostname) {
+ this.hostname = hostname;
+ }
+ public String toString() {
+ StringBuffer result = new StringBuffer();
+ result.append(splits.size());
+ result.append(" ");
+ result.append(hostname);
+ return result.toString();
+ }
+ }
+
+ List<String> readFile(String filename) throws IOException {
+ List<String> result = new ArrayList<String>(10000);
+ BufferedReader in = new BufferedReader(
+ new InputStreamReader(new FileInputStream(filename), Charsets.UTF_8));
+ String line = in.readLine();
+ while (line != null) {
+ result.add(line);
+ line = in.readLine();
+ }
+ in.close();
+ return result;
+ }
+
+ public TeraScheduler(String splitFilename,
+ String nodeFilename) throws IOException {
+ slotsPerHost = 4;
+ // get the hosts
+ Map<String, Host> hostIds = new HashMap<String,Host>();
+ for(String hostName: readFile(nodeFilename)) {
+ Host host = new Host(hostName);
+ hosts.add(host);
+ hostIds.put(hostName, host);
+ }
+ // read the blocks
+ List<String> splitLines = readFile(splitFilename);
+ splits = new Split[splitLines.size()];
+ remainingSplits = 0;
+ for(String line: splitLines) {
+ StringTokenizer itr = new StringTokenizer(line);
+ Split newSplit = new Split(itr.nextToken());
+ splits[remainingSplits++] = newSplit;
+ while (itr.hasMoreTokens()) {
+ Host host = hostIds.get(itr.nextToken());
+ newSplit.locations.add(host);
+ host.splits.add(newSplit);
+ }
+ }
+ }
+
+ public TeraScheduler(FileSplit[] realSplits,
+ Configuration conf) throws IOException {
+ this.realSplits = realSplits;
+ this.slotsPerHost = conf.getInt(TTConfig.TT_MAP_SLOTS, 4);
+ Map<String, Host> hostTable = new HashMap<String, Host>();
+ splits = new Split[realSplits.length];
+ for(FileSplit realSplit: realSplits) {
+ Split split = new Split(realSplit.getPath().toString());
+ splits[remainingSplits++] = split;
+ for(String hostname: realSplit.getLocations()) {
+ Host host = hostTable.get(hostname);
+ if (host == null) {
+ host = new Host(hostname);
+ hostTable.put(hostname, host);
+ hosts.add(host);
+ }
+ host.splits.add(split);
+ split.locations.add(host);
+ }
+ }
+ }
+
+ Host pickBestHost() {
+ Host result = null;
+ int splits = Integer.MAX_VALUE;
+ for(Host host: hosts) {
+ if (host.splits.size() < splits) {
+ result = host;
+ splits = host.splits.size();
+ }
+ }
+ if (result != null) {
+ hosts.remove(result);
+ LOG.debug("picking " + result);
+ }
+ return result;
+ }
+
+ void pickBestSplits(Host host) {
+ int tasksToPick = Math.min(slotsPerHost,
+ (int) Math.ceil((double) remainingSplits /
+ hosts.size()));
+ Split[] best = new Split[tasksToPick];
+ for(Split cur: host.splits) {
+ LOG.debug(" examine: " + cur.filename + " " + cur.locations.size());
+ int i = 0;
+ while (i < tasksToPick && best[i] != null &&
+ best[i].locations.size() <= cur.locations.size()) {
+ i += 1;
+ }
+ if (i < tasksToPick) {
+ for(int j = tasksToPick - 1; j > i; --j) {
+ best[j] = best[j-1];
+ }
+ best[i] = cur;
+ }
+ }
+ // for the chosen blocks, remove them from the other locations
+ for(int i=0; i < tasksToPick; ++i) {
+ if (best[i] != null) {
+ LOG.debug(" best: " + best[i].filename);
+ for (Host other: best[i].locations) {
+ other.splits.remove(best[i]);
+ }
+ best[i].locations.clear();
+ best[i].locations.add(host);
+ best[i].isAssigned = true;
+ remainingSplits -= 1;
+ }
+ }
+ // for the non-chosen blocks, remove this host
+ for(Split cur: host.splits) {
+ if (!cur.isAssigned) {
+ cur.locations.remove(host);
+ }
+ }
+ }
+
+ void solve() throws IOException {
+ Host host = pickBestHost();
+ while (host != null) {
+ pickBestSplits(host);
+ host = pickBestHost();
+ }
+ }
+
+ /**
+ * Solve the schedule and modify the FileSplit array to reflect the new
+ * schedule. It will move placed splits to front and unplacable splits
+ * to the end.
+ * @return a new list of FileSplits that are modified to have the
+ * best host as the only host.
+ * @throws IOException
+ */
+ public List<InputSplit> getNewFileSplits() throws IOException {
+ solve();
+ FileSplit[] result = new FileSplit[realSplits.length];
+ int left = 0;
+ int right = realSplits.length - 1;
+ for(int i=0; i < splits.length; ++i) {
+ if (splits[i].isAssigned) {
+ // copy the split and fix up the locations
+ ((TeraFileSplit) realSplits[i]).setLocations
+ (new String[]{splits[i].locations.get(0).hostname});
+ result[left++] = realSplits[i];
+ } else {
+ result[right--] = realSplits[i];
+ }
+ }
+ List<InputSplit> ret = new ArrayList<InputSplit>();
+ for (FileSplit fs : result) {
+ ret.add(fs);
+ }
+ return ret;
+ }
+
+ public static void main(String[] args) throws IOException {
+ TeraScheduler problem = new TeraScheduler("block-loc.txt", "nodes");
+ for(Host host: problem.hosts) {
+ System.out.println(host);
+ }
+ LOG.info("starting solve");
+ problem.solve();
+ List<Split> leftOvers = new ArrayList<Split>();
+ for(int i=0; i < problem.splits.length; ++i) {
+ if (problem.splits[i].isAssigned) {
+ System.out.println("sched: " + problem.splits[i]);
+ } else {
+ leftOvers.add(problem.splits[i]);
+ }
+ }
+ for(Split cur: leftOvers) {
+ System.out.println("left: " + cur);
+ }
+ System.out.println("left over: " + leftOvers.size());
+ LOG.info("done");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraSort.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraSort.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraSort.java
new file mode 100644
index 0000000..5a097f2
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraSort.java
@@ -0,0 +1,335 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples.terasort;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.net.URI;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.Partitioner;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * Generates the sampled split points, launches the job, and waits for it to
+ * finish.
+ * <p>
+ * To run the program:
+ * <b>bin/hadoop jar hadoop-*-examples.jar terasort in-dir out-dir</b>
+ */
+public class TeraSort extends Configured implements Tool {
+ private static final Log LOG = LogFactory.getLog(TeraSort.class);
+ static String SIMPLE_PARTITIONER = "mapreduce.terasort.simplepartitioner";
+ static String OUTPUT_REPLICATION = "mapreduce.terasort.output.replication";
+
+ /**
+ * A partitioner that splits text keys into roughly equal partitions
+ * in a global sorted order.
+ */
+ static class TotalOrderPartitioner extends Partitioner<Text,Text>
+ implements Configurable {
+ private TrieNode trie;
+ private Text[] splitPoints;
+ private Configuration conf;
+
+ /**
+ * A generic trie node
+ */
+ static abstract class TrieNode {
+ private int level;
+ TrieNode(int level) {
+ this.level = level;
+ }
+ abstract int findPartition(Text key);
+ abstract void print(PrintStream strm) throws IOException;
+ int getLevel() {
+ return level;
+ }
+ }
+
+ /**
+ * An inner trie node that contains 256 children based on the next
+ * character.
+ */
+ static class InnerTrieNode extends TrieNode {
+ private TrieNode[] child = new TrieNode[256];
+
+ InnerTrieNode(int level) {
+ super(level);
+ }
+ int findPartition(Text key) {
+ int level = getLevel();
+ if (key.getLength() <= level) {
+ return child[0].findPartition(key);
+ }
+ return child[key.getBytes()[level] & 0xff].findPartition(key);
+ }
+ void setChild(int idx, TrieNode child) {
+ this.child[idx] = child;
+ }
+ void print(PrintStream strm) throws IOException {
+ for(int ch=0; ch < 256; ++ch) {
+ for(int i = 0; i < 2*getLevel(); ++i) {
+ strm.print(' ');
+ }
+ strm.print(ch);
+ strm.println(" ->");
+ if (child[ch] != null) {
+ child[ch].print(strm);
+ }
+ }
+ }
+ }
+
+ /**
+ * A leaf trie node that does string compares to figure out where the given
+ * key belongs between lower..upper.
+ */
+ static class LeafTrieNode extends TrieNode {
+ int lower;
+ int upper;
+ Text[] splitPoints;
+ LeafTrieNode(int level, Text[] splitPoints, int lower, int upper) {
+ super(level);
+ this.splitPoints = splitPoints;
+ this.lower = lower;
+ this.upper = upper;
+ }
+ int findPartition(Text key) {
+ for(int i=lower; i<upper; ++i) {
+ if (splitPoints[i].compareTo(key) > 0) {
+ return i;
+ }
+ }
+ return upper;
+ }
+ void print(PrintStream strm) throws IOException {
+ for(int i = 0; i < 2*getLevel(); ++i) {
+ strm.print(' ');
+ }
+ strm.print(lower);
+ strm.print(", ");
+ strm.println(upper);
+ }
+ }
+
+
+ /**
+ * Read the cut points from the given sequence file.
+ * @param fs the file system
+ * @param p the path to read
+ * @param job the job config
+ * @return the strings to split the partitions on
+ * @throws IOException
+ */
+ private static Text[] readPartitions(FileSystem fs, Path p,
+ Configuration conf) throws IOException {
+ int reduces = conf.getInt(MRJobConfig.NUM_REDUCES, 1);
+ Text[] result = new Text[reduces - 1];
+ DataInputStream reader = fs.open(p);
+ for(int i=0; i < reduces - 1; ++i) {
+ result[i] = new Text();
+ result[i].readFields(reader);
+ }
+ reader.close();
+ return result;
+ }
+
+ /**
+ * Given a sorted set of cut points, build a trie that will find the correct
+ * partition quickly.
+ * @param splits the list of cut points
+ * @param lower the lower bound of partitions 0..numPartitions-1
+ * @param upper the upper bound of partitions 0..numPartitions-1
+ * @param prefix the prefix that we have already checked against
+ * @param maxDepth the maximum depth we will build a trie for
+ * @return the trie node that will divide the splits correctly
+ */
+ private static TrieNode buildTrie(Text[] splits, int lower, int upper,
+ Text prefix, int maxDepth) {
+ int depth = prefix.getLength();
+ if (depth >= maxDepth || lower == upper) {
+ return new LeafTrieNode(depth, splits, lower, upper);
+ }
+ InnerTrieNode result = new InnerTrieNode(depth);
+ Text trial = new Text(prefix);
+ // append an extra byte on to the prefix
+ trial.append(new byte[1], 0, 1);
+ int currentBound = lower;
+ for(int ch = 0; ch < 255; ++ch) {
+ trial.getBytes()[depth] = (byte) (ch + 1);
+ lower = currentBound;
+ while (currentBound < upper) {
+ if (splits[currentBound].compareTo(trial) >= 0) {
+ break;
+ }
+ currentBound += 1;
+ }
+ trial.getBytes()[depth] = (byte) ch;
+ result.child[ch] = buildTrie(splits, lower, currentBound, trial,
+ maxDepth);
+ }
+ // pick up the rest
+ trial.getBytes()[depth] = (byte) 255;
+ result.child[255] = buildTrie(splits, currentBound, upper, trial,
+ maxDepth);
+ return result;
+ }
+
+ public void setConf(Configuration conf) {
+ try {
+ FileSystem fs = FileSystem.getLocal(conf);
+ this.conf = conf;
+ Path partFile = new Path(TeraInputFormat.PARTITION_FILENAME);
+ splitPoints = readPartitions(fs, partFile, conf);
+ trie = buildTrie(splitPoints, 0, splitPoints.length, new Text(), 2);
+ } catch (IOException ie) {
+ throw new IllegalArgumentException("can't read paritions file", ie);
+ }
+ }
+
+ public Configuration getConf() {
+ return conf;
+ }
+
+ public TotalOrderPartitioner() {
+ }
+
+ public int getPartition(Text key, Text value, int numPartitions) {
+ return trie.findPartition(key);
+ }
+
+ }
+
+ /**
+ * A total order partitioner that assigns keys based on their first
+ * PREFIX_LENGTH bytes, assuming a flat distribution.
+ */
+ public static class SimplePartitioner extends Partitioner<Text, Text>
+ implements Configurable {
+ int prefixesPerReduce;
+ private static final int PREFIX_LENGTH = 3;
+ private Configuration conf = null;
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ prefixesPerReduce = (int) Math.ceil((1 << (8 * PREFIX_LENGTH)) /
+ (float) conf.getInt(MRJobConfig.NUM_REDUCES, 1));
+ }
+
+ public Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ public int getPartition(Text key, Text value, int numPartitions) {
+ byte[] bytes = key.getBytes();
+ int len = Math.min(PREFIX_LENGTH, key.getLength());
+ int prefix = 0;
+ for(int i=0; i < len; ++i) {
+ prefix = (prefix << 8) | (0xff & bytes[i]);
+ }
+ return prefix / prefixesPerReduce;
+ }
+ }
+
+ public static boolean getUseSimplePartitioner(JobContext job) {
+ return job.getConfiguration().getBoolean(SIMPLE_PARTITIONER, false);
+ }
+
+ public static void setUseSimplePartitioner(Job job, boolean value) {
+ job.getConfiguration().setBoolean(SIMPLE_PARTITIONER, value);
+ }
+
+ public static int getOutputReplication(JobContext job) {
+ return job.getConfiguration().getInt(OUTPUT_REPLICATION, 1);
+ }
+
+ public static void setOutputReplication(Job job, int value) {
+ job.getConfiguration().setInt(OUTPUT_REPLICATION, value);
+ }
+
+ public int run(String[] args) throws Exception {
+ if (args.length != 2) {
+ System.err.println("Invalid no. of arguments provided");
+ System.err.println("Usage: terasort <input-dir> <output-dir>");
+ return -1;
+ }
+
+ LOG.info("starting");
+ Job job = Job.getInstance(getConf());
+ Path inputDir = new Path(args[0]);
+ Path outputDir = new Path(args[1]);
+ boolean useSimplePartitioner = getUseSimplePartitioner(job);
+ TeraInputFormat.setInputPaths(job, inputDir);
+ FileOutputFormat.setOutputPath(job, outputDir);
+ job.setJobName("TeraSort");
+ job.setJarByClass(TeraSort.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(Text.class);
+ job.setInputFormatClass(TeraInputFormat.class);
+ job.setOutputFormatClass(TeraOutputFormat.class);
+ if (useSimplePartitioner) {
+ job.setPartitionerClass(SimplePartitioner.class);
+ } else {
+ long start = System.currentTimeMillis();
+ Path partitionFile = new Path(outputDir,
+ TeraInputFormat.PARTITION_FILENAME);
+ URI partitionUri = new URI(partitionFile.toString() +
+ "#" + TeraInputFormat.PARTITION_FILENAME);
+ try {
+ TeraInputFormat.writePartitionFile(job, partitionFile);
+ } catch (Throwable e) {
+ LOG.error(e.getMessage());
+ return -1;
+ }
+ job.addCacheFile(partitionUri);
+ long end = System.currentTimeMillis();
+ System.out.println("Spent " + (end - start) + "ms computing partitions.");
+ job.setPartitionerClass(TotalOrderPartitioner.class);
+ }
+
+ job.getConfiguration().setInt("dfs.replication", getOutputReplication(job));
+ TeraOutputFormat.setFinalSync(job, true);
+ int ret = job.waitForCompletion(true) ? 0 : 1;
+ LOG.info("done");
+ return ret;
+ }
+
+ /**
+ * @param args
+ */
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(), new TeraSort(), args);
+ System.exit(res);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraValidate.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraValidate.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraValidate.java
new file mode 100644
index 0000000..24c3145
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraValidate.java
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples.terasort;
+
+import java.io.IOException;
+import java.util.zip.Checksum;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.PureJavaCrc32;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * Generate 1 mapper per a file that checks to make sure the keys
+ * are sorted within each file. The mapper also generates
+ * "$file:begin", first key and "$file:end", last key. The reduce verifies that
+ * all of the start/end items are in order.
+ * Any output from the reduce is problem report.
+ * <p>
+ * To run the program:
+ * <b>bin/hadoop jar hadoop-*-examples.jar teravalidate out-dir report-dir</b>
+ * <p>
+ * If there is any output, something is wrong and the output of the reduce
+ * will have the problem report.
+ */
+public class TeraValidate extends Configured implements Tool {
+ private static final Text ERROR = new Text("error");
+ private static final Text CHECKSUM = new Text("checksum");
+
+ private static String textifyBytes(Text t) {
+ BytesWritable b = new BytesWritable();
+ b.set(t.getBytes(), 0, t.getLength());
+ return b.toString();
+ }
+
+ static class ValidateMapper extends Mapper<Text,Text,Text,Text> {
+ private Text lastKey;
+ private String filename;
+ private Unsigned16 checksum = new Unsigned16();
+ private Unsigned16 tmp = new Unsigned16();
+ private Checksum crc32 = new PureJavaCrc32();
+
+ /**
+ * Get the final part of the input name
+ * @param split the input split
+ * @return the "part-r-00000" for the input
+ */
+ private String getFilename(FileSplit split) {
+ return split.getPath().getName();
+ }
+
+ public void map(Text key, Text value, Context context)
+ throws IOException, InterruptedException {
+ if (lastKey == null) {
+ FileSplit fs = (FileSplit) context.getInputSplit();
+ filename = getFilename(fs);
+ context.write(new Text(filename + ":begin"), key);
+ lastKey = new Text();
+ } else {
+ if (key.compareTo(lastKey) < 0) {
+ context.write(ERROR, new Text("misorder in " + filename +
+ " between " + textifyBytes(lastKey) +
+ " and " + textifyBytes(key)));
+ }
+ }
+ // compute the crc of the key and value and add it to the sum
+ crc32.reset();
+ crc32.update(key.getBytes(), 0, key.getLength());
+ crc32.update(value.getBytes(), 0, value.getLength());
+ tmp.set(crc32.getValue());
+ checksum.add(tmp);
+ lastKey.set(key);
+ }
+
+ public void cleanup(Context context)
+ throws IOException, InterruptedException {
+ if (lastKey != null) {
+ context.write(new Text(filename + ":end"), lastKey);
+ context.write(CHECKSUM, new Text(checksum.toString()));
+ }
+ }
+ }
+
+ /**
+ * Check the boundaries between the output files by making sure that the
+ * boundary keys are always increasing.
+ * Also passes any error reports along intact.
+ */
+ static class ValidateReducer extends Reducer<Text,Text,Text,Text> {
+ private boolean firstKey = true;
+ private Text lastKey = new Text();
+ private Text lastValue = new Text();
+ public void reduce(Text key, Iterable<Text> values,
+ Context context) throws IOException, InterruptedException {
+ if (ERROR.equals(key)) {
+ for (Text val : values) {
+ context.write(key, val);
+ }
+ } else if (CHECKSUM.equals(key)) {
+ Unsigned16 tmp = new Unsigned16();
+ Unsigned16 sum = new Unsigned16();
+ for (Text val : values) {
+ tmp.set(val.toString());
+ sum.add(tmp);
+ }
+ context.write(CHECKSUM, new Text(sum.toString()));
+ } else {
+ Text value = values.iterator().next();
+ if (firstKey) {
+ firstKey = false;
+ } else {
+ if (value.compareTo(lastValue) < 0) {
+ context.write(ERROR,
+ new Text("bad key partitioning:\n file " +
+ lastKey + " key " +
+ textifyBytes(lastValue) +
+ "\n file " + key + " key " +
+ textifyBytes(value)));
+ }
+ }
+ lastKey.set(key);
+ lastValue.set(value);
+ }
+ }
+
+ }
+
+ private static void usage() throws IOException {
+ System.err.println("teravalidate <out-dir> <report-dir>");
+ }
+
+ public int run(String[] args) throws Exception {
+ Job job = Job.getInstance(getConf());
+ if (args.length != 2) {
+ usage();
+ return 1;
+ }
+ TeraInputFormat.setInputPaths(job, new Path(args[0]));
+ FileOutputFormat.setOutputPath(job, new Path(args[1]));
+ job.setJobName("TeraValidate");
+ job.setJarByClass(TeraValidate.class);
+ job.setMapperClass(ValidateMapper.class);
+ job.setReducerClass(ValidateReducer.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(Text.class);
+ // force a single reducer
+ job.setNumReduceTasks(1);
+ // force a single split
+ FileInputFormat.setMinInputSplitSize(job, Long.MAX_VALUE);
+ job.setInputFormatClass(TeraInputFormat.class);
+ return job.waitForCompletion(true) ? 0 : 1;
+ }
+
+ /**
+ * @param args
+ */
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(), new TeraValidate(), args);
+ System.exit(res);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/Unsigned16.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/Unsigned16.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/Unsigned16.java
new file mode 100644
index 0000000..1335bf4
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/Unsigned16.java
@@ -0,0 +1,297 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples.terasort;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.Writable;
+
+/**
+ * An unsigned 16 byte integer class that supports addition, multiplication,
+ * and left shifts.
+ */
+class Unsigned16 implements Writable {
+ private long hi8;
+ private long lo8;
+
+ public Unsigned16() {
+ hi8 = 0;
+ lo8 = 0;
+ }
+
+ public Unsigned16(long l) {
+ hi8 = 0;
+ lo8 = l;
+ }
+
+ public Unsigned16(Unsigned16 other) {
+ hi8 = other.hi8;
+ lo8 = other.lo8;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof Unsigned16) {
+ Unsigned16 other = (Unsigned16) o;
+ return other.hi8 == hi8 && other.lo8 == lo8;
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return (int) lo8;
+ }
+
+ /**
+ * Parse a hex string
+ * @param s the hex string
+ */
+ public Unsigned16(String s) throws NumberFormatException {
+ set(s);
+ }
+
+ /**
+ * Set the number from a hex string
+ * @param s the number in hexadecimal
+ * @throws NumberFormatException if the number is invalid
+ */
+ public void set(String s) throws NumberFormatException {
+ hi8 = 0;
+ lo8 = 0;
+ final long lastDigit = 0xfl << 60;
+ for (int i = 0; i < s.length(); ++i) {
+ int digit = getHexDigit(s.charAt(i));
+ if ((lastDigit & hi8) != 0) {
+ throw new NumberFormatException(s + " overflowed 16 bytes");
+ }
+ hi8 <<= 4;
+ hi8 |= (lo8 & lastDigit) >>> 60;
+ lo8 <<= 4;
+ lo8 |= digit;
+ }
+ }
+
+ /**
+ * Set the number to a given long.
+ * @param l the new value, which is treated as an unsigned number
+ */
+ public void set(long l) {
+ lo8 = l;
+ hi8 = 0;
+ }
+
+ /**
+ * Map a hexadecimal character into a digit.
+ * @param ch the character
+ * @return the digit from 0 to 15
+ * @throws NumberFormatException
+ */
+ private static int getHexDigit(char ch) throws NumberFormatException {
+ if (ch >= '0' && ch <= '9') {
+ return ch - '0';
+ }
+ if (ch >= 'a' && ch <= 'f') {
+ return ch - 'a' + 10;
+ }
+ if (ch >= 'A' && ch <= 'F') {
+ return ch - 'A' + 10;
+ }
+ throw new NumberFormatException(ch + " is not a valid hex digit");
+ }
+
+ private static final Unsigned16 TEN = new Unsigned16(10);
+
+ public static Unsigned16 fromDecimal(String s) throws NumberFormatException {
+ Unsigned16 result = new Unsigned16();
+ Unsigned16 tmp = new Unsigned16();
+ for(int i=0; i < s.length(); i++) {
+ char ch = s.charAt(i);
+ if (ch < '0' || ch > '9') {
+ throw new NumberFormatException(ch + " not a valid decimal digit");
+ }
+ int digit = ch - '0';
+ result.multiply(TEN);
+ tmp.set(digit);
+ result.add(tmp);
+ }
+ return result;
+ }
+
+ /**
+ * Return the number as a hex string.
+ */
+ public String toString() {
+ if (hi8 == 0) {
+ return Long.toHexString(lo8);
+ } else {
+ StringBuilder result = new StringBuilder();
+ result.append(Long.toHexString(hi8));
+ String loString = Long.toHexString(lo8);
+ for(int i=loString.length(); i < 16; ++i) {
+ result.append('0');
+ }
+ result.append(loString);
+ return result.toString();
+ }
+ }
+
+ /**
+ * Get a given byte from the number.
+ * @param b the byte to get with 0 meaning the most significant byte
+ * @return the byte or 0 if b is outside of 0..15
+ */
+ public byte getByte(int b) {
+ if (b >= 0 && b < 16) {
+ if (b < 8) {
+ return (byte) (hi8 >> (56 - 8*b));
+ } else {
+ return (byte) (lo8 >> (120 - 8*b));
+ }
+ }
+ return 0;
+ }
+
+ /**
+ * Get the hexadecimal digit at the given position.
+ * @param p the digit position to get with 0 meaning the most significant
+ * @return the character or '0' if p is outside of 0..31
+ */
+ public char getHexDigit(int p) {
+ byte digit = getByte(p / 2);
+ if (p % 2 == 0) {
+ digit >>>= 4;
+ }
+ digit &= 0xf;
+ if (digit < 10) {
+ return (char) ('0' + digit);
+ } else {
+ return (char) ('A' + digit - 10);
+ }
+ }
+
+ /**
+ * Get the high 8 bytes as a long.
+ */
+ public long getHigh8() {
+ return hi8;
+ }
+
+ /**
+ * Get the low 8 bytes as a long.
+ */
+ public long getLow8() {
+ return lo8;
+ }
+
+ /**
+ * Multiple the current number by a 16 byte unsigned integer. Overflow is not
+ * detected and the result is the low 16 bytes of the result. The numbers
+ * are divided into 32 and 31 bit chunks so that the product of two chucks
+ * fits in the unsigned 63 bits of a long.
+ * @param b the other number
+ */
+ void multiply(Unsigned16 b) {
+ // divide the left into 4 32 bit chunks
+ long[] left = new long[4];
+ left[0] = lo8 & 0xffffffffl;
+ left[1] = lo8 >>> 32;
+ left[2] = hi8 & 0xffffffffl;
+ left[3] = hi8 >>> 32;
+ // divide the right into 5 31 bit chunks
+ long[] right = new long[5];
+ right[0] = b.lo8 & 0x7fffffffl;
+ right[1] = (b.lo8 >>> 31) & 0x7fffffffl;
+ right[2] = (b.lo8 >>> 62) + ((b.hi8 & 0x1fffffffl) << 2);
+ right[3] = (b.hi8 >>> 29) & 0x7fffffffl;
+ right[4] = (b.hi8 >>> 60);
+ // clear the cur value
+ set(0);
+ Unsigned16 tmp = new Unsigned16();
+ for(int l=0; l < 4; ++l) {
+ for (int r=0; r < 5; ++r) {
+ long prod = left[l] * right[r];
+ if (prod != 0) {
+ int off = l*32 + r*31;
+ tmp.set(prod);
+ tmp.shiftLeft(off);
+ add(tmp);
+ }
+ }
+ }
+ }
+
+ /**
+ * Add the given number into the current number.
+ * @param b the other number
+ */
+ public void add(Unsigned16 b) {
+ long sumHi;
+ long sumLo;
+ long reshibit, hibit0, hibit1;
+
+ sumHi = hi8 + b.hi8;
+
+ hibit0 = (lo8 & 0x8000000000000000L);
+ hibit1 = (b.lo8 & 0x8000000000000000L);
+ sumLo = lo8 + b.lo8;
+ reshibit = (sumLo & 0x8000000000000000L);
+ if ((hibit0 & hibit1) != 0 | ((hibit0 ^ hibit1) != 0 && reshibit == 0))
+ sumHi++; /* add carry bit */
+ hi8 = sumHi;
+ lo8 = sumLo;
+ }
+
+ /**
+ * Shift the number a given number of bit positions. The number is the low
+ * order bits of the result.
+ * @param bits the bit positions to shift by
+ */
+ public void shiftLeft(int bits) {
+ if (bits != 0) {
+ if (bits < 64) {
+ hi8 <<= bits;
+ hi8 |= (lo8 >>> (64 - bits));
+ lo8 <<= bits;
+ } else if (bits < 128) {
+ hi8 = lo8 << (bits - 64);
+ lo8 = 0;
+ } else {
+ hi8 = 0;
+ lo8 = 0;
+ }
+ }
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ hi8 = in.readLong();
+ lo8 = in.readLong();
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeLong(hi8);
+ out.writeLong(lo8);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/test/java/org/apache/tez/mapreduce/examples/terasort/TestTeraSort.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/mapreduce/examples/terasort/TestTeraSort.java b/tez-tests/src/test/java/org/apache/tez/mapreduce/examples/terasort/TestTeraSort.java
new file mode 100644
index 0000000..97b6538
--- /dev/null
+++ b/tez-tests/src/test/java/org/apache/tez/mapreduce/examples/terasort/TestTeraSort.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.mapreduce.examples.terasort;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.HadoopTestCase;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.tez.mapreduce.examples.terasort.TeraGen;
+import org.apache.tez.mapreduce.examples.terasort.TeraSort;
+import org.apache.tez.mapreduce.examples.terasort.TeraValidate;
+
+import org.junit.Ignore;
+
+@Ignore
+public class TestTeraSort extends HadoopTestCase {
+
+ public TestTeraSort()
+ throws IOException {
+ super(CLUSTER_MR, DFS_FS, 1, 1);
+ }
+
+ protected void tearDown() throws Exception {
+ getFileSystem().delete(new Path(TEST_DIR), true);
+ super.tearDown();
+ }
+
+ // Input/Output paths for sort
+ private static final String TEST_DIR =
+ new File(System.getProperty("test.build.data", "/tmp"), "org/apache/tez/mapreduce/examples/terasort")
+ .getAbsolutePath();
+ private static final Path SORT_INPUT_PATH = new Path(TEST_DIR, "sortin");
+ private static final Path SORT_OUTPUT_PATH = new Path(TEST_DIR, "sortout");
+ private static final Path TERA_OUTPUT_PATH = new Path(TEST_DIR, "validate");
+ private static final String NUM_ROWS = "100";
+
+ private void runTeraGen(Configuration conf, Path sortInput)
+ throws Exception {
+ String[] genArgs = {NUM_ROWS, sortInput.toString()};
+
+ // Run TeraGen
+ assertEquals(ToolRunner.run(conf, new TeraGen(), genArgs), 0);
+ }
+
+ private void runTeraSort(Configuration conf,
+ Path sortInput, Path sortOutput) throws Exception {
+
+ // Setup command-line arguments to 'sort'
+ String[] sortArgs = {sortInput.toString(), sortOutput.toString()};
+
+ // Run Sort
+ assertEquals(ToolRunner.run(conf, new TeraSort(), sortArgs), 0);
+ }
+
+ private void runTeraValidator(Configuration job,
+ Path sortOutput, Path valOutput)
+ throws Exception {
+ String[] svArgs = {sortOutput.toString(), valOutput.toString()};
+
+ // Run Tera-Validator
+ assertEquals(ToolRunner.run(job, new TeraValidate(), svArgs), 0);
+ }
+
+ public void testTeraSort() throws Exception {
+ // Run TeraGen to generate input for 'terasort'
+ runTeraGen(createJobConf(), SORT_INPUT_PATH);
+
+ // Run terasort
+ runTeraSort(createJobConf(), SORT_INPUT_PATH, SORT_OUTPUT_PATH);
+
+ // Run tera-validator to check if sort worked correctly
+ runTeraValidator(createJobConf(), SORT_OUTPUT_PATH,
+ TERA_OUTPUT_PATH);
+ }
+
+}
[04/10] TEZ-1055. Rename tez-mapreduce-examples to tez-examples
(Hitesh Shah via bikas)
Posted by bi...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/GroupByOrderByMRRTest.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/GroupByOrderByMRRTest.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/GroupByOrderByMRRTest.java
new file mode 100644
index 0000000..939bea0
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/GroupByOrderByMRRTest.java
@@ -0,0 +1,247 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples;
+
+import java.io.IOException;
+import java.util.StringTokenizer;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.MRTezClient;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.mapreduce.hadoop.MRJobConfig;
+import org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil;
+
+/**
+ * Simple example that does a GROUP BY ORDER BY in an MRR job
+ * Consider a query such as
+ * Select DeptName, COUNT(*) as cnt FROM EmployeeTable
+ * GROUP BY DeptName ORDER BY cnt;
+ *
+ * i.e. List all departments with count of employees in each department
+ * and ordered based on department's employee count.
+ *
+ * Requires an Input file containing 2 strings per line in format of
+ * <EmployeeName> <DeptName>
+ *
+ * For example, use the following:
+ *
+ * #/bin/bash
+ *
+ * i=1000000
+ * j=1000
+ *
+ * id=0
+ * while [[ "$id" -ne "$i" ]]
+ * do
+ * id=`expr $id + 1`
+ * deptId=`expr $RANDOM % $j + 1`
+ * deptName=`echo "ibase=10;obase=16;$deptId" | bc`
+ * echo "$id O$deptName"
+ * done
+ *
+ */
+public class GroupByOrderByMRRTest extends Configured implements Tool {
+
+ private static final Log LOG = LogFactory.getLog(GroupByOrderByMRRTest.class);
+
+ /**
+ * Mapper takes in a single line as input containing
+ * employee name and department name and then
+ * emits department name with count of 1
+ */
+ public static class MyMapper
+ extends Mapper<Object, Text, Text, IntWritable> {
+
+ private final static IntWritable one = new IntWritable(1);
+ private final static Text word = new Text();
+
+ public void map(Object key, Text value, Context context
+ ) throws IOException, InterruptedException {
+ StringTokenizer itr = new StringTokenizer(value.toString());
+ String empName = "";
+ String deptName = "";
+ if (itr.hasMoreTokens()) {
+ empName = itr.nextToken();
+ if (itr.hasMoreTokens()) {
+ deptName = itr.nextToken();
+ }
+ if (!empName.isEmpty()
+ && !deptName.isEmpty()) {
+ word.set(deptName);
+ context.write(word, one);
+ }
+ }
+ }
+ }
+
+ /**
+ * Intermediate reducer aggregates the total count per department.
+ * It takes department name and count as input and emits the final
+ * count per department name.
+ */
+ public static class MyGroupByReducer
+ extends Reducer<Text, IntWritable, IntWritable, Text> {
+ private IntWritable result = new IntWritable();
+
+ public void reduce(Text key, Iterable<IntWritable> values,
+ Context context
+ ) throws IOException, InterruptedException {
+
+ int sum = 0;
+ for (IntWritable val : values) {
+ sum += val.get();
+ }
+ result.set(sum);
+ context.write(result, key);
+ }
+ }
+
+ /**
+ * Shuffle ensures ordering based on count of employees per department
+ * hence the final reducer is a no-op and just emits the department name
+ * with the employee count per department.
+ */
+ public static class MyOrderByNoOpReducer
+ extends Reducer<IntWritable, Text, Text, IntWritable> {
+
+ public void reduce(IntWritable key, Iterable<Text> values,
+ Context context
+ ) throws IOException, InterruptedException {
+ for (Text word : values) {
+ context.write(word, key);
+ }
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Configuration conf = getConf();
+
+ // Configure intermediate reduces
+ conf.setInt(MRJobConfig.MRR_INTERMEDIATE_STAGES, 1);
+
+ // Set reducer class for intermediate reduce
+ conf.setClass(MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1,
+ "mapreduce.job.reduce.class"), MyGroupByReducer.class, Reducer.class);
+ // Set reducer output key class
+ conf.setClass(MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1,
+ "mapreduce.map.output.key.class"), IntWritable.class, Object.class);
+ // Set reducer output value class
+ conf.setClass(MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1,
+ "mapreduce.map.output.value.class"), Text.class, Object.class);
+ conf.setInt(MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1,
+ "mapreduce.job.reduces"), 2);
+
+ String[] otherArgs = new GenericOptionsParser(conf, args).
+ getRemainingArgs();
+ if (otherArgs.length != 2) {
+ System.err.println("Usage: groupbyorderbymrrtest <in> <out>");
+ ToolRunner.printGenericCommandUsage(System.err);
+ return 2;
+ }
+
+ @SuppressWarnings("deprecation")
+ Job job = new Job(conf, "groupbyorderbymrrtest");
+
+ job.setJarByClass(GroupByOrderByMRRTest.class);
+
+ // Configure map
+ job.setMapperClass(MyMapper.class);
+ job.setMapOutputKeyClass(Text.class);
+ job.setMapOutputValueClass(IntWritable.class);
+
+ // Configure reduce
+ job.setReducerClass(MyOrderByNoOpReducer.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+ job.setNumReduceTasks(1);
+
+ FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
+ FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
+
+ job.submit();
+ JobID jobId = job.getJobID();
+ ApplicationId appId = TypeConverter.toYarn(jobId).getAppId();
+
+ DAGClient dagClient = MRTezClient.getDAGClient(appId, new TezConfiguration(conf), null);
+ DAGStatus dagStatus;
+ String[] vNames = { "initialmap" , "ireduce1" , "finalreduce" };
+ while (true) {
+ dagStatus = dagClient.getDAGStatus(null);
+ if(dagStatus.getState() == DAGStatus.State.RUNNING ||
+ dagStatus.getState() == DAGStatus.State.SUCCEEDED ||
+ dagStatus.getState() == DAGStatus.State.FAILED ||
+ dagStatus.getState() == DAGStatus.State.KILLED ||
+ dagStatus.getState() == DAGStatus.State.ERROR) {
+ break;
+ }
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ // continue;
+ }
+ }
+
+ while (dagStatus.getState() == DAGStatus.State.RUNNING) {
+ try {
+ ExampleDriver.printDAGStatus(dagClient, vNames);
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ // continue;
+ }
+ dagStatus = dagClient.getDAGStatus(null);
+ } catch (TezException e) {
+ LOG.fatal("Failed to get application progress. Exiting");
+ return -1;
+ }
+ }
+
+ ExampleDriver.printDAGStatus(dagClient, vNames);
+ LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
+ return dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1;
+ }
+
+ public static void main(String[] args) throws Exception {
+ Configuration configuration = new Configuration();
+ GroupByOrderByMRRTest groupByOrderByMRRTest = new GroupByOrderByMRRTest();
+ int status = ToolRunner.run(configuration, groupByOrderByMRRTest, args);
+ System.exit(status);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/IntersectDataGen.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/IntersectDataGen.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/IntersectDataGen.java
new file mode 100644
index 0000000..711240c
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/IntersectDataGen.java
@@ -0,0 +1,350 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.tez.client.TezClient;
+import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.mapreduce.output.MROutput;
+import org.apache.tez.mapreduce.processor.SimpleMRProcessor;
+import org.apache.tez.runtime.api.ProcessorContext;
+import org.apache.tez.runtime.library.api.KeyValueWriter;
+
+import com.google.common.base.Preconditions;
+
+public class IntersectDataGen extends Configured implements Tool {
+
+ private static final Log LOG = LogFactory.getLog(IntersectDataGen.class);
+
+ private static final String STREAM_OUTPUT_NAME = "streamoutput";
+ private static final String HASH_OUTPUT_NAME = "hashoutput";
+ private static final String EXPECTED_OUTPUT_NAME = "expectedoutput";
+
+ public static void main(String[] args) throws Exception {
+ IntersectDataGen dataGen = new IntersectDataGen();
+ int status = ToolRunner.run(new Configuration(), dataGen, args);
+ System.exit(status);
+ }
+
+ private static void printUsage() {
+ System.err
+ .println("Usage: "
+ + "intersectdatagen <outPath1> <path1Size> <outPath2> <path2Size> <expectedResultPath> <numTasks>");
+ ;
+ ToolRunner.printGenericCommandUsage(System.err);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Configuration conf = getConf();
+ String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+ int result = validateArgs(otherArgs);
+ if (result != 0) {
+ return result;
+ }
+ return execute(otherArgs);
+ }
+
+ public int run(Configuration conf, String[] args, TezClient tezSession) throws Exception {
+ setConf(conf);
+ String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+ int result = validateArgs(otherArgs);
+ if (result != 0) {
+ return result;
+ }
+ return execute(otherArgs, tezSession);
+ }
+
+ private int validateArgs(String[] otherArgs) {
+ if (otherArgs.length != 6) {
+ printUsage();
+ return 2;
+ }
+ return 0;
+ }
+
+ private int execute(String [] args) throws TezException, IOException, InterruptedException {
+ TezConfiguration tezConf = new TezConfiguration(getConf());
+ TezClient tezSession = null;
+ try {
+ tezSession = createTezSession(tezConf);
+ return execute(args, tezConf, tezSession);
+ } finally {
+ if (tezSession != null) {
+ tezSession.stop();
+ }
+ }
+ }
+
+ private int execute(String[] args, TezClient tezSession) throws IOException, TezException,
+ InterruptedException {
+ TezConfiguration tezConf = new TezConfiguration(getConf());
+ return execute(args, tezConf, tezSession);
+ }
+
+ private TezClient createTezSession(TezConfiguration tezConf) throws TezException, IOException {
+ TezClient tezSession = new TezClient("IntersectDataGenSession", tezConf);
+ tezSession.start();
+ return tezSession;
+ }
+
+ private int execute(String[] args, TezConfiguration tezConf, TezClient tezSession)
+ throws IOException, TezException, InterruptedException {
+ LOG.info("Running IntersectDataGen");
+
+ UserGroupInformation.setConfiguration(tezConf);
+
+ String outDir1 = args[0];
+ long outDir1Size = Long.parseLong(args[1]);
+ String outDir2 = args[2];
+ long outDir2Size = Long.parseLong(args[3]);
+ String expectedOutputDir = args[4];
+ int numTasks = Integer.parseInt(args[5]);
+
+ Path largeOutPath = null;
+ Path smallOutPath = null;
+ long largeOutSize = 0;
+ long smallOutSize = 0;
+
+ if (outDir1Size >= outDir2Size) {
+ largeOutPath = new Path(outDir1);
+ largeOutSize = outDir1Size;
+ smallOutPath = new Path(outDir2);
+ smallOutSize = outDir2Size;
+ } else {
+ largeOutPath = new Path(outDir2);
+ largeOutSize = outDir2Size;
+ smallOutPath = new Path(outDir1);
+ smallOutSize = outDir1Size;
+ }
+
+ Path expectedOutputPath = new Path(expectedOutputDir);
+
+ // Verify output path existence
+ FileSystem fs = FileSystem.get(tezConf);
+ int res = 0;
+ res = checkOutputDirectory(fs, largeOutPath) + checkOutputDirectory(fs, smallOutPath)
+ + checkOutputDirectory(fs, expectedOutputPath);
+ if (res != 0) {
+ return 3;
+ }
+
+ if (numTasks <= 0) {
+ System.err.println("NumTasks must be > 0");
+ return 4;
+ }
+
+ DAG dag = createDag(tezConf, largeOutPath, smallOutPath, expectedOutputPath, numTasks,
+ largeOutSize, smallOutSize);
+
+ tezSession.waitTillReady();
+ DAGClient dagClient = tezSession.submitDAG(dag);
+ DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(null);
+ if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
+ LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics());
+ return -1;
+ }
+ return 0;
+
+ }
+
+ private DAG createDag(TezConfiguration tezConf, Path largeOutPath, Path smallOutPath,
+ Path expectedOutputPath, int numTasks, long largeOutSize, long smallOutSize)
+ throws IOException {
+
+ long largeOutSizePerTask = largeOutSize / numTasks;
+ long smallOutSizePerTask = smallOutSize / numTasks;
+
+ DAG dag = new DAG("IntersectDataGen");
+
+ Vertex genDataVertex = new Vertex("datagen", new ProcessorDescriptor(
+ GenDataProcessor.class.getName()).setUserPayload(
+ new UserPayload(GenDataProcessor.createConfiguration(largeOutSizePerTask, smallOutSizePerTask))), numTasks);
+ genDataVertex.addDataSink(STREAM_OUTPUT_NAME,
+ MROutput.createConfigurer(new Configuration(tezConf),
+ TextOutputFormat.class, largeOutPath.toUri().toString()).create());
+ genDataVertex.addDataSink(HASH_OUTPUT_NAME,
+ MROutput.createConfigurer(new Configuration(tezConf),
+ TextOutputFormat.class, smallOutPath.toUri().toString()).create());
+ genDataVertex.addDataSink(EXPECTED_OUTPUT_NAME,
+ MROutput.createConfigurer(new Configuration(tezConf),
+ TextOutputFormat.class, expectedOutputPath.toUri().toString()).create());
+
+ dag.addVertex(genDataVertex);
+
+ return dag;
+ }
+
+ public static class GenDataProcessor extends SimpleMRProcessor {
+
+ private static final Log LOG = LogFactory.getLog(GenDataProcessor.class);
+
+ long streamOutputFileSize;
+ long hashOutputFileSize;
+ float overlapApprox = 0.2f;
+
+ public GenDataProcessor(ProcessorContext context) {
+ super(context);
+ }
+
+ public static byte[] createConfiguration(long streamOutputFileSize, long hashOutputFileSize)
+ throws IOException {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(bos);
+ dos.writeLong(streamOutputFileSize);
+ dos.writeLong(hashOutputFileSize);
+ dos.close();
+ bos.close();
+ return bos.toByteArray();
+ }
+
+ @Override
+ public void initialize() throws Exception {
+ byte[] payload = getContext().getUserPayload().getPayload();
+ ByteArrayInputStream bis = new ByteArrayInputStream(payload);
+ DataInputStream dis = new DataInputStream(bis);
+ streamOutputFileSize = dis.readLong();
+ hashOutputFileSize = dis.readLong();
+ LOG.info("Initialized with largeFileTargetSize=" + streamOutputFileSize
+ + ", smallFileTragetSize=" + hashOutputFileSize);
+ dis.close();
+ bis.close();
+ }
+
+ @Override
+ public void run() throws Exception {
+ Preconditions.checkState(getInputs().size() == 0);
+ Preconditions.checkState(getOutputs().size() == 3);
+
+ KeyValueWriter streamOutputWriter = (KeyValueWriter) getOutputs().get(STREAM_OUTPUT_NAME)
+ .getWriter();
+ KeyValueWriter hashOutputWriter = (KeyValueWriter) getOutputs().get(HASH_OUTPUT_NAME)
+ .getWriter();
+ KeyValueWriter expectedOutputWriter = (KeyValueWriter) getOutputs().get(EXPECTED_OUTPUT_NAME)
+ .getWriter();
+
+ float fileSizeFraction = hashOutputFileSize / (float) streamOutputFileSize;
+ Preconditions.checkState(fileSizeFraction > 0.0f && fileSizeFraction <= 1.0f);
+ int mod = 1;
+ int extraKeysMod = 0;
+ if (fileSizeFraction > overlapApprox) {
+ // Common keys capped by overlap. Additional ones required in the hashFile.
+ mod = (int) (1 / overlapApprox);
+ extraKeysMod = (int) (1 / (fileSizeFraction - overlapApprox));
+ } else {
+ // All keys in hashFile must exist in stream file.
+ mod = (int) (1 / fileSizeFraction);
+ }
+ LOG.info("Using mod=" + mod + ", extraKeysMod=" + extraKeysMod);
+
+ long count = 0;
+ long sizeLarge = 0;
+ long sizeSmall = 0;
+ long numLargeFileKeys = 0;
+ long numSmallFileKeys = 0;
+ long numExpectedKeys = 0;
+ while (sizeLarge < streamOutputFileSize) {
+ String str = createOverlapString(13, count);
+ Text text = new Text(str);
+ int size = text.getLength();
+ streamOutputWriter.write(text, NullWritable.get());
+ sizeLarge += size;
+ numLargeFileKeys++;
+ if (count % mod == 0) {
+ hashOutputWriter.write(text, NullWritable.get());
+ sizeSmall += size;
+ numSmallFileKeys++;
+ expectedOutputWriter.write(text, NullWritable.get());
+ numExpectedKeys++;
+ }
+ if (extraKeysMod != 0 && count % extraKeysMod == 0) {
+ String nStr = createNonOverlaptring(13, count);
+ Text nText = new Text(nStr);
+ hashOutputWriter.write(nText, NullWritable.get());
+ sizeSmall += nText.getLength();
+ numSmallFileKeys++;
+ }
+ count++;
+ }
+ LOG.info("OutputStats: " + "largeFileNumKeys=" + numLargeFileKeys + ", smallFileNumKeys="
+ + numSmallFileKeys + ", expFileNumKeys=" + numExpectedKeys + ", largeFileSize="
+ + sizeLarge + ", smallFileSize=" + sizeSmall);
+ }
+
+ private String createOverlapString(int size, long count) {
+ StringBuilder sb = new StringBuilder();
+ Random random = new Random();
+ for (int i = 0; i < size; i++) {
+ int r = Math.abs(random.nextInt()) % 26;
+ // Random a-z followed by the count
+ sb.append((char) (97 + r));
+ }
+ sb.append("_").append(getContext().getTaskIndex()).append("_").append(count);
+ return sb.toString();
+ }
+
+ private String createNonOverlaptring(int size, long count) {
+ StringBuilder sb = new StringBuilder();
+ Random random = new Random();
+ for (int i = 0; i < size; i++) {
+ int r = Math.abs(random.nextInt()) % 26;
+ // Random A-Z followed by the count
+ sb.append((char) (65 + r));
+ }
+ sb.append("_").append(getContext().getTaskIndex()).append("_").append(count);
+ return sb.toString();
+ }
+
+ }
+
+ private int checkOutputDirectory(FileSystem fs, Path path) throws IOException {
+ if (fs.exists(path)) {
+ System.err.println("Output directory: " + path + " already exists");
+ return 2;
+ }
+ return 0;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/IntersectExample.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/IntersectExample.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/IntersectExample.java
new file mode 100644
index 0000000..da18014
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/IntersectExample.java
@@ -0,0 +1,281 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples;
+
+import java.io.IOException;
+import java.util.HashSet;
+
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.tez.client.TezClient;
+import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.Edge;
+import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.mapreduce.input.MRInput;
+import org.apache.tez.mapreduce.output.MROutput;
+import org.apache.tez.mapreduce.processor.SimpleMRProcessor;
+import org.apache.tez.runtime.api.LogicalInput;
+import org.apache.tez.runtime.api.LogicalOutput;
+import org.apache.tez.runtime.api.Reader;
+import org.apache.tez.runtime.api.ProcessorContext;
+import org.apache.tez.runtime.library.api.KeyValueReader;
+import org.apache.tez.runtime.library.api.KeyValueWriter;
+import org.apache.tez.runtime.library.conf.UnorderedPartitionedKVEdgeConfigurer;
+import org.apache.tez.runtime.library.partitioner.HashPartitioner;
+import org.apache.tez.runtime.library.processor.SimpleProcessor;
+
+import com.google.common.base.Preconditions;
+
+public class IntersectExample extends Configured implements Tool {
+
+ private static final Log LOG = LogFactory.getLog(IntersectExample.class);
+
+ public static void main(String[] args) throws Exception {
+ IntersectExample intersect = new IntersectExample();
+ int status = ToolRunner.run(new Configuration(), intersect, args);
+ System.exit(status);
+ }
+
+ private static void printUsage() {
+ System.err.println("Usage: " + "intersect <file1> <file2> <numPartitions> <outPath>");
+ ToolRunner.printGenericCommandUsage(System.err);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Configuration conf = getConf();
+ String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+ int result = validateArgs(otherArgs);
+ if (result != 0) {
+ return result;
+ }
+ return execute(otherArgs);
+ }
+
+ public int run(Configuration conf, String[] args, TezClient tezSession) throws Exception {
+ setConf(conf);
+ String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+ int result = validateArgs(otherArgs);
+ if (result != 0) {
+ return result;
+ }
+ return execute(otherArgs, tezSession);
+ }
+
+ private int validateArgs(String[] otherArgs) {
+ if (otherArgs.length != 4) {
+ printUsage();
+ return 2;
+ }
+ return 0;
+ }
+
+ private int execute(String[] args) throws TezException, IOException, InterruptedException {
+ TezConfiguration tezConf = new TezConfiguration(getConf());
+ TezClient tezSession = null;
+ try {
+ tezSession = createTezSession(tezConf);
+ return execute(args, tezConf, tezSession);
+ } finally {
+ if (tezSession != null) {
+ tezSession.stop();
+ }
+ }
+ }
+
+ private int execute(String[] args, TezClient tezSession) throws IOException, TezException,
+ InterruptedException {
+ TezConfiguration tezConf = new TezConfiguration(getConf());
+ return execute(args, tezConf, tezSession);
+ }
+
+ private TezClient createTezSession(TezConfiguration tezConf) throws TezException, IOException {
+ TezClient tezSession = new TezClient("IntersectExampleSession", tezConf);
+ tezSession.start();
+ return tezSession;
+ }
+
+ private int execute(String[] args, TezConfiguration tezConf, TezClient tezSession)
+ throws IOException, TezException, InterruptedException {
+ LOG.info("Running IntersectExample");
+
+ UserGroupInformation.setConfiguration(tezConf);
+
+ String streamInputDir = args[0];
+ String hashInputDir = args[1];
+ int numPartitions = Integer.parseInt(args[2]);
+ String outputDir = args[3];
+
+ Path streamInputPath = new Path(streamInputDir);
+ Path hashInputPath = new Path(hashInputDir);
+ Path outputPath = new Path(outputDir);
+
+ // Verify output path existence
+ FileSystem fs = FileSystem.get(tezConf);
+ if (fs.exists(outputPath)) {
+ System.err.println("Output directory: " + outputDir + " already exists");
+ return 3;
+ }
+ if (numPartitions <= 0) {
+ System.err.println("NumPartitions must be > 0");
+ return 4;
+ }
+
+ DAG dag = createDag(tezConf, streamInputPath, hashInputPath, outputPath, numPartitions);
+
+ tezSession.waitTillReady();
+ DAGClient dagClient = tezSession.submitDAG(dag);
+ DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(null);
+ if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
+ LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics());
+ return -1;
+ }
+ return 0;
+
+ }
+
+ private DAG createDag(TezConfiguration tezConf, Path streamPath, Path hashPath, Path outPath,
+ int numPartitions) throws IOException {
+ DAG dag = new DAG("IntersectExample");
+
+ // Configuration for intermediate output - shared by Vertex1 and Vertex2
+ // This should only be setting selective keys from the underlying conf. Fix after there's a
+ // better mechanism to configure the IOs.
+
+ UnorderedPartitionedKVEdgeConfigurer edgeConf =
+ UnorderedPartitionedKVEdgeConfigurer
+ .newBuilder(Text.class.getName(), NullWritable.class.getName(),
+ HashPartitioner.class.getName()).build();
+
+ // Change the way resources are setup - no MRHelpers
+ Vertex streamFileVertex = new Vertex("partitioner1", new ProcessorDescriptor(
+ ForwardingProcessor.class.getName())).addDataSource(
+ "streamfile",
+ MRInput
+ .createConfigurer(new Configuration(tezConf), TextInputFormat.class,
+ streamPath.toUri().toString()).groupSplits(false).create());
+
+ Vertex hashFileVertex = new Vertex("partitioner2", new ProcessorDescriptor(
+ ForwardingProcessor.class.getName())).addDataSource(
+ "hashfile",
+ MRInput
+ .createConfigurer(new Configuration(tezConf), TextInputFormat.class,
+ hashPath.toUri().toString()).groupSplits(false).create());
+
+ Vertex intersectVertex = new Vertex("intersect", new ProcessorDescriptor(
+ IntersectProcessor.class.getName()), numPartitions).addDataSink("finalOutput",
+ MROutput.createConfigurer(new Configuration(tezConf),
+ TextOutputFormat.class, outPath.toUri().toString()).create());
+
+ Edge e1 = new Edge(streamFileVertex, intersectVertex, edgeConf.createDefaultEdgeProperty());
+
+ Edge e2 = new Edge(hashFileVertex, intersectVertex, edgeConf.createDefaultEdgeProperty());
+
+ dag.addVertex(streamFileVertex).addVertex(hashFileVertex).addVertex(intersectVertex)
+ .addEdge(e1).addEdge(e2);
+ return dag;
+ }
+
+ // private void obtainTokens(Credentials credentials, Path... paths) throws IOException {
+ // TokenCache.obtainTokensForNamenodes(credentials, paths, getConf());
+ // }
+
+ /**
+ * Reads key-values from the source and forwards the value as the key for the output
+ */
+ public static class ForwardingProcessor extends SimpleProcessor {
+ public ForwardingProcessor(ProcessorContext context) {
+ super(context);
+ }
+
+ @Override
+ public void run() throws Exception {
+ Preconditions.checkState(getInputs().size() == 1);
+ Preconditions.checkState(getOutputs().size() == 1);
+ LogicalInput input = getInputs().values().iterator().next();
+ Reader rawReader = input.getReader();
+ Preconditions.checkState(rawReader instanceof KeyValueReader);
+ LogicalOutput output = getOutputs().values().iterator().next();
+
+ KeyValueReader reader = (KeyValueReader) rawReader;
+ KeyValueWriter writer = (KeyValueWriter) output.getWriter();
+
+ while (reader.next()) {
+ Object val = reader.getCurrentValue();
+ writer.write(val, NullWritable.get());
+ }
+ }
+ }
+
+ public static class IntersectProcessor extends SimpleMRProcessor {
+
+ public IntersectProcessor(ProcessorContext context) {
+ super(context);
+ }
+
+ @Override
+ public void run() throws Exception {
+ Preconditions.checkState(getInputs().size() == 2);
+ Preconditions.checkState(getOutputs().size() == 1);
+ LogicalInput streamInput = getInputs().get("partitioner1");
+ LogicalInput hashInput = getInputs().get("partitioner2");
+ Reader rawStreamReader = streamInput.getReader();
+ Reader rawHashReader = hashInput.getReader();
+ Preconditions.checkState(rawStreamReader instanceof KeyValueReader);
+ Preconditions.checkState(rawHashReader instanceof KeyValueReader);
+ LogicalOutput lo = getOutputs().values().iterator().next();
+ Preconditions.checkState(lo instanceof MROutput);
+ MROutput output = (MROutput) lo;
+ KeyValueWriter writer = output.getWriter();
+
+ KeyValueReader hashKvReader = (KeyValueReader) rawHashReader;
+ Set<Text> keySet = new HashSet<Text>();
+ while (hashKvReader.next()) {
+ keySet.add(new Text((Text) hashKvReader.getCurrentKey()));
+ }
+
+ KeyValueReader streamKvReader = (KeyValueReader) rawStreamReader;
+ while (streamKvReader.next()) {
+ Text key = (Text) streamKvReader.getCurrentKey();
+ if (keySet.contains(key)) {
+ writer.write(key, NullWritable.get());
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/IntersectValidate.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/IntersectValidate.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/IntersectValidate.java
new file mode 100644
index 0000000..c5328c7
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/IntersectValidate.java
@@ -0,0 +1,257 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.tez.client.TezClient;
+import org.apache.tez.common.counters.TezCounter;
+import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.Edge;
+import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.api.client.StatusGetOpts;
+import org.apache.tez.mapreduce.examples.IntersectExample.ForwardingProcessor;
+import org.apache.tez.mapreduce.input.MRInput;
+import org.apache.tez.runtime.api.LogicalInput;
+import org.apache.tez.runtime.api.Reader;
+import org.apache.tez.runtime.api.ProcessorContext;
+import org.apache.tez.runtime.library.api.KeyValuesReader;
+import org.apache.tez.runtime.library.conf.OrderedPartitionedKVEdgeConfigurer;
+import org.apache.tez.runtime.library.partitioner.HashPartitioner;
+import org.apache.tez.runtime.library.processor.SimpleProcessor;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
+
+public class IntersectValidate extends Configured implements Tool {
+ private static final Log LOG = LogFactory.getLog(IntersectExample.class);
+
+ private static final String LHS_INPUT_NAME = "lhsfile";
+ private static final String RHS_INPUT_NAME = "rhsfile";
+
+ private static final String COUNTER_GROUP_NAME = "INTERSECT_VALIDATE";
+ private static final String MISSING_KEY_COUNTER_NAME = "MISSING_KEY_EXISTS";
+
+ public static void main(String[] args) throws Exception {
+ IntersectValidate validate = new IntersectValidate();
+ int status = ToolRunner.run(new Configuration(), validate, args);
+ System.exit(status);
+ }
+
+ private static void printUsage() {
+ System.err.println("Usage: " + "intersectvalidate <path1> <path2>");
+ ToolRunner.printGenericCommandUsage(System.err);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Configuration conf = getConf();
+ String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+ int result = validateArgs(otherArgs);
+ if (result != 0) {
+ return result;
+ }
+ return execute(otherArgs);
+ }
+
+ public int run(Configuration conf, String[] args, TezClient tezSession) throws Exception {
+ setConf(conf);
+ String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+ int result = validateArgs(otherArgs);
+ if (result != 0) {
+ return result;
+ }
+ return execute(otherArgs, tezSession);
+ }
+
+ private int validateArgs(String[] otherArgs) {
+ if (otherArgs.length != 3 && otherArgs.length != 2) {
+ printUsage();
+ return 2;
+ }
+ return 0;
+ }
+
+ private int execute(String[] args) throws TezException, IOException, InterruptedException {
+ TezConfiguration tezConf = new TezConfiguration(getConf());
+ TezClient tezSession = null;
+ try {
+ tezSession = createTezSession(tezConf);
+ return execute(args, tezConf, tezSession);
+ } finally {
+ if (tezSession != null) {
+ tezSession.stop();
+ }
+ }
+ }
+
+ private int execute(String[] args, TezClient tezSession) throws IOException, TezException,
+ InterruptedException {
+ TezConfiguration tezConf = new TezConfiguration(getConf());
+ return execute(args, tezConf, tezSession);
+ }
+
+ private TezClient createTezSession(TezConfiguration tezConf) throws TezException, IOException {
+ TezClient tezSession = new TezClient("IntersectValidateSession", tezConf);
+ tezSession.start();
+ return tezSession;
+ }
+
+ private int execute(String[] args, TezConfiguration tezConf, TezClient tezSession)
+ throws IOException, TezException, InterruptedException {
+ LOG.info("Running IntersectValidate");
+ UserGroupInformation.setConfiguration(tezConf);
+
+ String lhsDir = args[0];
+ String rhsDir = args[1];
+ int numPartitions = 1;
+ if (args.length == 3) {
+ numPartitions = Integer.parseInt(args[2]);
+ }
+
+ if (numPartitions <= 0) {
+ System.err.println("NumPartitions must be > 0");
+ return 4;
+ }
+
+ Path lhsPath = new Path(lhsDir);
+ Path rhsPath = new Path(rhsDir);
+
+ DAG dag = createDag(tezConf, lhsPath, rhsPath, numPartitions);
+
+ tezSession.waitTillReady();
+ DAGClient dagClient = tezSession.submitDAG(dag);
+ DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(null);
+ if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
+ LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics());
+ return -1;
+ } else {
+ dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
+ TezCounter counter = dagStatus.getDAGCounters().findCounter(COUNTER_GROUP_NAME,
+ MISSING_KEY_COUNTER_NAME);
+ if (counter == null) {
+ LOG.info("Unable to determing equality");
+ return -2;
+ } else {
+ if (counter.getValue() != 0) {
+ LOG.info("Validate failed. The two sides are not equivalent");
+ return -3;
+ } else {
+ LOG.info("Validation successful. The two sides are equivalent");
+ return 0;
+ }
+ }
+ }
+ }
+
+ private DAG createDag(TezConfiguration tezConf, Path lhs, Path rhs, int numPartitions)
+ throws IOException {
+ DAG dag = new DAG("IntersectValidate");
+
+ // Configuration for intermediate output - shared by Vertex1 and Vertex2
+ // This should only be setting selective keys from the underlying conf. Fix after there's a
+ // better mechanism to configure the IOs.
+ OrderedPartitionedKVEdgeConfigurer edgeConf = OrderedPartitionedKVEdgeConfigurer
+ .newBuilder(Text.class.getName(), NullWritable.class.getName(),
+ HashPartitioner.class.getName()).build();
+
+ Vertex lhsVertex = new Vertex(LHS_INPUT_NAME, new ProcessorDescriptor(
+ ForwardingProcessor.class.getName())).addDataSource("lhs",
+ MRInput
+ .createConfigurer(new Configuration(tezConf), TextInputFormat.class,
+ lhs.toUri().toString()).groupSplits(false).create());
+
+ Vertex rhsVertex = new Vertex(RHS_INPUT_NAME, new ProcessorDescriptor(
+ ForwardingProcessor.class.getName())).addDataSource("rhs",
+ MRInput
+ .createConfigurer(new Configuration(tezConf), TextInputFormat.class,
+ rhs.toUri().toString()).groupSplits(false).create());
+
+ Vertex intersectValidateVertex = new Vertex("intersectvalidate", new ProcessorDescriptor(
+ IntersectValidateProcessor.class.getName()), numPartitions);
+
+ Edge e1 = new Edge(lhsVertex, intersectValidateVertex, edgeConf.createDefaultEdgeProperty());
+ Edge e2 = new Edge(rhsVertex, intersectValidateVertex, edgeConf.createDefaultEdgeProperty());
+
+ dag.addVertex(lhsVertex).addVertex(rhsVertex).addVertex(intersectValidateVertex).addEdge(e1)
+ .addEdge(e2);
+ return dag;
+ }
+
+ public static class IntersectValidateProcessor extends SimpleProcessor {
+
+ private static final Log LOG = LogFactory.getLog(IntersectValidateProcessor.class);
+
+ public IntersectValidateProcessor(ProcessorContext context) {
+ super(context);
+ }
+
+ @Override
+ public void run() throws Exception {
+ Preconditions.checkState(getInputs().size() == 2);
+ Preconditions.checkState(getOutputs().size() == 0);
+ LogicalInput lhsInput = getInputs().get(LHS_INPUT_NAME);
+ LogicalInput rhsInput = getInputs().get(RHS_INPUT_NAME);
+ Reader lhsReaderRaw = lhsInput.getReader();
+ Reader rhsReaderRaw = rhsInput.getReader();
+ Preconditions.checkState(lhsReaderRaw instanceof KeyValuesReader);
+ Preconditions.checkState(rhsReaderRaw instanceof KeyValuesReader);
+ KeyValuesReader lhsReader = (KeyValuesReader) lhsReaderRaw;
+ KeyValuesReader rhsReader = (KeyValuesReader) rhsReaderRaw;
+
+ TezCounter lhsMissingKeyCounter = getContext().getCounters().findCounter(COUNTER_GROUP_NAME,
+ MISSING_KEY_COUNTER_NAME);
+
+ while (lhsReader.next()) {
+ if (rhsReader.next()) {
+ if (!lhsReader.getCurrentKey().equals(rhsReader.getCurrentKey())) {
+ LOG.info("MismatchedKeys: " + "lhs=" + lhsReader.getCurrentKey() + ", rhs=" + rhsReader.getCurrentKey());
+ lhsMissingKeyCounter.increment(1);
+ }
+ } else {
+ lhsMissingKeyCounter.increment(1);
+ LOG.info("ExtraKey in lhs: " + lhsReader.getClass());
+ break;
+ }
+ }
+ if (rhsReader.next()) {
+ lhsMissingKeyCounter.increment(1);
+ LOG.info("ExtraKey in rhs: " + lhsReader.getClass());
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Join.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Join.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Join.java
new file mode 100644
index 0000000..a3a65ec
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Join.java
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapred.ClusterStatus;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.join.CompositeInputFormat;
+import org.apache.hadoop.mapreduce.lib.join.TupleWritable;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * Given a set of sorted datasets keyed with the same class and yielding
+ * equal partitions, it is possible to effect a join of those datasets
+ * prior to the map. The example facilitates the same.
+ *
+ * To run: bin/hadoop jar build/hadoop-examples.jar join
+ * [-r <i>reduces</i>]
+ * [-inFormat <i>input format class</i>]
+ * [-outFormat <i>output format class</i>]
+ * [-outKey <i>output key class</i>]
+ * [-outValue <i>output value class</i>]
+ * [-joinOp <inner|outer|override>]
+ * [<i>in-dir</i>]* <i>in-dir</i> <i>out-dir</i>
+ */
+public class Join extends Configured implements Tool {
+ public static final String REDUCES_PER_HOST = "mapreduce.join.reduces_per_host";
+ static int printUsage() {
+ System.out.println("join [-r <reduces>] " +
+ "[-inFormat <input format class>] " +
+ "[-outFormat <output format class>] " +
+ "[-outKey <output key class>] " +
+ "[-outValue <output value class>] " +
+ "[-joinOp <inner|outer|override>] " +
+ "[input]* <input> <output>");
+ ToolRunner.printGenericCommandUsage(System.out);
+ return 2;
+ }
+
+ /**
+ * The main driver for sort program.
+ * Invoke this method to submit the map/reduce job.
+ * @throws IOException When there is communication problems with the
+ * job tracker.
+ */
+ @SuppressWarnings("unchecked")
+ public int run(String[] args) throws Exception {
+ Configuration conf = getConf();
+ JobClient client = new JobClient(conf);
+ ClusterStatus cluster = client.getClusterStatus();
+ int num_reduces = (int) (cluster.getMaxReduceTasks() * 0.9);
+ String join_reduces = conf.get(REDUCES_PER_HOST);
+ if (join_reduces != null) {
+ num_reduces = cluster.getTaskTrackers() *
+ Integer.parseInt(join_reduces);
+ }
+ Job job = new Job(conf);
+ job.setJobName("join");
+ job.setJarByClass(Sort.class);
+
+ job.setMapperClass(Mapper.class);
+ job.setReducerClass(Reducer.class);
+
+ Class<? extends InputFormat> inputFormatClass =
+ SequenceFileInputFormat.class;
+ Class<? extends OutputFormat> outputFormatClass =
+ SequenceFileOutputFormat.class;
+ Class<? extends WritableComparable> outputKeyClass = BytesWritable.class;
+ Class<? extends Writable> outputValueClass = TupleWritable.class;
+ String op = "inner";
+ List<String> otherArgs = new ArrayList<String>();
+ for(int i=0; i < args.length; ++i) {
+ try {
+ if ("-r".equals(args[i])) {
+ num_reduces = Integer.parseInt(args[++i]);
+ } else if ("-inFormat".equals(args[i])) {
+ inputFormatClass =
+ Class.forName(args[++i]).asSubclass(InputFormat.class);
+ } else if ("-outFormat".equals(args[i])) {
+ outputFormatClass =
+ Class.forName(args[++i]).asSubclass(OutputFormat.class);
+ } else if ("-outKey".equals(args[i])) {
+ outputKeyClass =
+ Class.forName(args[++i]).asSubclass(WritableComparable.class);
+ } else if ("-outValue".equals(args[i])) {
+ outputValueClass =
+ Class.forName(args[++i]).asSubclass(Writable.class);
+ } else if ("-joinOp".equals(args[i])) {
+ op = args[++i];
+ } else {
+ otherArgs.add(args[i]);
+ }
+ } catch (NumberFormatException except) {
+ System.out.println("ERROR: Integer expected instead of " + args[i]);
+ return printUsage();
+ } catch (ArrayIndexOutOfBoundsException except) {
+ System.out.println("ERROR: Required parameter missing from " +
+ args[i-1]);
+ return printUsage(); // exits
+ }
+ }
+
+ // Set user-supplied (possibly default) job configs
+ job.setNumReduceTasks(num_reduces);
+
+ if (otherArgs.size() < 2) {
+ System.out.println("ERROR: Wrong number of parameters: ");
+ return printUsage();
+ }
+
+ FileOutputFormat.setOutputPath(job,
+ new Path(otherArgs.remove(otherArgs.size() - 1)));
+ List<Path> plist = new ArrayList<Path>(otherArgs.size());
+ for (String s : otherArgs) {
+ plist.add(new Path(s));
+ }
+
+ job.setInputFormatClass(CompositeInputFormat.class);
+ job.getConfiguration().set(CompositeInputFormat.JOIN_EXPR,
+ CompositeInputFormat.compose(op, inputFormatClass,
+ plist.toArray(new Path[0])));
+ job.setOutputFormatClass(outputFormatClass);
+
+ job.setOutputKeyClass(outputKeyClass);
+ job.setOutputValueClass(outputValueClass);
+
+ Date startTime = new Date();
+ System.out.println("Job started: " + startTime);
+ int ret = job.waitForCompletion(true) ? 0 : 1 ;
+ Date end_time = new Date();
+ System.out.println("Job ended: " + end_time);
+ System.out.println("The job took " +
+ (end_time.getTime() - startTime.getTime()) /1000 + " seconds.");
+ return ret;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(), new Join(), args);
+ System.exit(res);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
new file mode 100644
index 0000000..b22a4a8
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
@@ -0,0 +1,757 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.mapreduce.examples;
+
+import java.io.IOException;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Partitioner;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.hadoop.mapreduce.security.TokenCache;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.util.ClassUtil;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.client.TezClientUtils;
+import org.apache.tez.client.TezClient;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.DataSourceDescriptor;
+import org.apache.tez.dag.api.Edge;
+import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.mapreduce.hadoop.MRHelpers;
+import org.apache.tez.mapreduce.hadoop.MRInputHelpers;
+import org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil;
+import org.apache.tez.mapreduce.input.MRInputLegacy;
+import org.apache.tez.mapreduce.output.MROutputLegacy;
+import org.apache.tez.mapreduce.processor.map.MapProcessor;
+import org.apache.tez.mapreduce.processor.reduce.ReduceProcessor;
+import org.apache.tez.runtime.library.conf.OrderedPartitionedKVEdgeConfigurer;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.tez.runtime.library.partitioner.HashPartitioner;
+
+/**
+ * Dummy class for testing MR framefork. Sleeps for a defined period
+ * of time in mapper and reducer. Generates fake input for map / reduce
+ * jobs. Note that generated number of input pairs is in the order
+ * of <code>numMappers * mapSleepTime / 100</code>, so the job uses
+ * some disk space.
+ */
+public class MRRSleepJob extends Configured implements Tool {
+
+ private static final Log LOG = LogFactory.getLog(MRRSleepJob.class);
+
+ public static final String MAP_SLEEP_COUNT = "mrr.sleepjob.map.sleep.count";
+ public static final String REDUCE_SLEEP_COUNT =
+ "mrr.sleepjob.reduce.sleep.count";
+ public static final String MAP_SLEEP_TIME = "mrr.sleepjob.map.sleep.time";
+ public static final String REDUCE_SLEEP_TIME =
+ "mrr.sleepjob.reduce.sleep.time";
+ public static final String IREDUCE_SLEEP_COUNT =
+ "mrr.sleepjob.ireduce.sleep.count";
+ public static final String IREDUCE_SLEEP_TIME =
+ "mrr.sleepjob.ireduce.sleep.time";
+ public static final String IREDUCE_STAGES_COUNT =
+ "mrr.sleepjob.ireduces.stages.count";
+ public static final String IREDUCE_TASKS_COUNT =
+ "mrr.sleepjob.ireduces.tasks.count";
+
+ // Flags to inject failures
+ public static final String MAP_THROW_ERROR = "mrr.sleepjob.map.throw.error";
+ public static final String MAP_FATAL_ERROR = "mrr.sleepjob.map.fatal.error";
+ public static final String MAP_ERROR_TASK_IDS =
+ "mrr.sleepjob.map.error.task.ids";
+
+ public static class MRRSleepJobPartitioner extends
+ Partitioner<IntWritable, IntWritable> {
+ public int getPartition(IntWritable k, IntWritable v, int numPartitions) {
+ return k.get() % numPartitions;
+ }
+ }
+
+ public static class EmptySplit extends InputSplit implements Writable {
+ public void write(DataOutput out) throws IOException { }
+ public void readFields(DataInput in) throws IOException { }
+ public long getLength() { return 0L; }
+ public String[] getLocations() { return new String[0]; }
+ }
+
+ public static class SleepInputFormat
+ extends InputFormat<IntWritable,IntWritable> {
+
+ public List<InputSplit> getSplits(JobContext jobContext) {
+ List<InputSplit> ret = new ArrayList<InputSplit>();
+ int numSplits = jobContext.getConfiguration().
+ getInt(MRJobConfig.NUM_MAPS, 1);
+ for (int i = 0; i < numSplits; ++i) {
+ ret.add(new EmptySplit());
+ }
+ return ret;
+ }
+
+ public RecordReader<IntWritable,IntWritable> createRecordReader(
+ InputSplit ignored, TaskAttemptContext taskContext)
+ throws IOException {
+ Configuration conf = taskContext.getConfiguration();
+
+ final int count = conf.getInt(MAP_SLEEP_COUNT, 1);
+ if (count < 0) {
+ throw new IOException("Invalid map count: " + count);
+ }
+
+ int totalIReduces = conf.getInt(IREDUCE_STAGES_COUNT, 1);
+
+ int reduceTasks = totalIReduces == 0?
+ taskContext.getNumReduceTasks() :
+ conf.getInt(IREDUCE_TASKS_COUNT, 1);
+ int sleepCount = totalIReduces == 0?
+ conf.getInt(REDUCE_SLEEP_COUNT,1) :
+ conf.getInt(IREDUCE_SLEEP_COUNT,1);
+ final int emitPerMapTask = sleepCount * reduceTasks;
+
+ return new RecordReader<IntWritable,IntWritable>() {
+ private int records = 0;
+ private int emitCount = 0;
+ private IntWritable key = null;
+ private IntWritable value = null;
+
+ public void initialize(InputSplit split, TaskAttemptContext context) {
+ }
+
+ public boolean nextKeyValue()
+ throws IOException {
+ if (count == 0) {
+ return false;
+ }
+ key = new IntWritable();
+ key.set(emitCount);
+ int emit = emitPerMapTask / count;
+ if ((emitPerMapTask) % count > records) {
+ ++emit;
+ }
+ emitCount += emit;
+ value = new IntWritable();
+ value.set(emit);
+ return records++ < count;
+ }
+ public IntWritable getCurrentKey() { return key; }
+ public IntWritable getCurrentValue() { return value; }
+ public void close() throws IOException { }
+ public float getProgress() throws IOException {
+ return count == 0 ? 100 : records / ((float)count);
+ }
+ };
+ }
+ }
+
+ public static class SleepMapper
+ extends Mapper<IntWritable, IntWritable, IntWritable, IntWritable> {
+ private long mapSleepDuration = 100;
+ private int mapSleepCount = 1;
+ private int count = 0;
+ private String vertexName;
+ private boolean throwError = false;
+ private boolean throwFatal = false;
+ private boolean finalAttempt = false;
+
+ protected void setup(Context context)
+ throws IOException, InterruptedException {
+ Configuration conf = context.getConfiguration();
+ this.mapSleepCount =
+ conf.getInt(MAP_SLEEP_COUNT, mapSleepCount);
+ this.mapSleepDuration = mapSleepCount == 0 ? 0 :
+ conf.getLong(MAP_SLEEP_TIME , 100) / mapSleepCount;
+ vertexName = conf.get(
+ org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);
+
+ TaskAttemptID taId = context.getTaskAttemptID();
+
+ String[] taskIds = conf.getStrings(MAP_ERROR_TASK_IDS);
+ if (taId.getId()+1 >= context.getMaxMapAttempts()) {
+ finalAttempt = true;
+ }
+ boolean found = false;
+ if (taskIds != null) {
+ if (taskIds.length == 1 && taskIds[0].equals("*")) {
+ found = true;
+ }
+ if (!found) {
+ for (String taskId : taskIds) {
+ if (Integer.valueOf(taskId).intValue() ==
+ taId.getTaskID().getId()) {
+ found = true;
+ break;
+ }
+ }
+ }
+ }
+ if (found) {
+ if (!finalAttempt) {
+ throwError = conf.getBoolean(MAP_THROW_ERROR, false);
+ }
+ throwFatal = conf.getBoolean(MAP_FATAL_ERROR, false);
+ }
+ }
+
+ public void map(IntWritable key, IntWritable value, Context context
+ ) throws IOException, InterruptedException {
+ //it is expected that every map processes mapSleepCount number of records.
+ try {
+ LOG.info("Reading in " + vertexName
+ + " taskid " + context.getTaskAttemptID().getTaskID().getId()
+ + " key " + key.get());
+ LOG.info("Sleeping in InitialMap"
+ + ", vertexName=" + vertexName
+ + ", taskAttemptId=" + context.getTaskAttemptID()
+ + ", mapSleepDuration=" + mapSleepDuration
+ + ", mapSleepCount=" + mapSleepCount
+ + ", sleepLeft="
+ + (mapSleepDuration * (mapSleepCount - count)));
+ context.setStatus("Sleeping... (" +
+ (mapSleepDuration * (mapSleepCount - count)) + ") ms left");
+ if ((mapSleepCount - count) > 0) {
+ Thread.sleep(mapSleepDuration);
+ }
+ if (throwError || throwFatal) {
+ throw new IOException("Throwing a simulated error from map");
+ }
+ }
+ catch (InterruptedException ex) {
+ throw (IOException)new IOException(
+ "Interrupted while sleeping").initCause(ex);
+ }
+ ++count;
+ // output reduceSleepCount * numReduce number of random values, so that
+ // each reducer will get reduceSleepCount number of keys.
+ int k = key.get();
+ for (int i = 0; i < value.get(); ++i) {
+ LOG.info("Writing in " + vertexName
+ + " taskid " + context.getTaskAttemptID().getTaskID().getId()
+ + " key " + (k+i) + " value 1");
+ context.write(new IntWritable(k + i), new IntWritable(1));
+ }
+ }
+ }
+
+ public static class ISleepReducer
+ extends Reducer<IntWritable, IntWritable, IntWritable, IntWritable> {
+ private long iReduceSleepDuration = 100;
+ private int iReduceSleepCount = 1;
+ private int count = 0;
+ private String vertexName;
+
+ protected void setup(Context context)
+ throws IOException, InterruptedException {
+ Configuration conf = context.getConfiguration();
+ this.iReduceSleepCount =
+ conf.getInt(IREDUCE_SLEEP_COUNT, iReduceSleepCount);
+ this.iReduceSleepDuration = iReduceSleepCount == 0 ? 0 :
+ conf.getLong(IREDUCE_SLEEP_TIME , 100) / iReduceSleepCount;
+ vertexName = conf.get(
+ org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);
+ }
+
+ public void reduce(IntWritable key, Iterable<IntWritable> values,
+ Context context)
+ throws IOException, InterruptedException {
+ try {
+ LOG.info("Reading in " + vertexName
+ + " taskid " + context.getTaskAttemptID().getTaskID().getId()
+ + " key " + key.get());
+
+ LOG.info("Sleeping in IntermediateReduce"
+ + ", vertexName=" + vertexName
+ + ", taskAttemptId=" + context.getTaskAttemptID()
+ + ", iReduceSleepDuration=" + iReduceSleepDuration
+ + ", iReduceSleepCount=" + iReduceSleepCount
+ + ", sleepLeft="
+ + (iReduceSleepDuration * (iReduceSleepCount - count)));
+ context.setStatus("Sleeping... (" +
+ (iReduceSleepDuration * (iReduceSleepCount - count)) + ") ms left");
+ if ((iReduceSleepCount - count) > 0) {
+ Thread.sleep(iReduceSleepDuration);
+ }
+ }
+ catch (InterruptedException ex) {
+ throw (IOException)new IOException(
+ "Interrupted while sleeping").initCause(ex);
+ }
+ ++count;
+ // output reduceSleepCount * numReduce number of random values, so that
+ // each reducer will get reduceSleepCount number of keys.
+ int k = key.get();
+ for (IntWritable value : values) {
+ for (int i = 0; i < value.get(); ++i) {
+ LOG.info("Writing in " + vertexName
+ + " taskid " + context.getTaskAttemptID().getTaskID().getId()
+ + " key " + (k+i) + " value 1");
+ context.write(new IntWritable(k + i), new IntWritable(1));
+ }
+ }
+ }
+ }
+
+ public static class SleepReducer
+ extends Reducer<IntWritable, IntWritable, NullWritable, NullWritable> {
+ private long reduceSleepDuration = 100;
+ private int reduceSleepCount = 1;
+ private int count = 0;
+ private String vertexName;
+
+ protected void setup(Context context)
+ throws IOException, InterruptedException {
+ Configuration conf = context.getConfiguration();
+ this.reduceSleepCount =
+ conf.getInt(REDUCE_SLEEP_COUNT, reduceSleepCount);
+ this.reduceSleepDuration = reduceSleepCount == 0 ? 0 :
+ conf.getLong(REDUCE_SLEEP_TIME , 100) / reduceSleepCount;
+ vertexName = conf.get(
+ org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);
+ }
+
+ public void reduce(IntWritable key, Iterable<IntWritable> values,
+ Context context)
+ throws IOException {
+ try {
+ LOG.info("Reading in " + vertexName
+ + " taskid " + context.getTaskAttemptID().getTaskID().getId()
+ + " key " + key.get());
+ LOG.info("Sleeping in FinalReduce"
+ + ", vertexName=" + vertexName
+ + ", taskAttemptId=" + context.getTaskAttemptID()
+ + ", reduceSleepDuration=" + reduceSleepDuration
+ + ", reduceSleepCount=" + reduceSleepCount
+ + ", sleepLeft="
+ + (reduceSleepDuration * (reduceSleepCount - count)));
+ context.setStatus("Sleeping... (" +
+ (reduceSleepDuration * (reduceSleepCount - count)) + ") ms left");
+ if ((reduceSleepCount - count) > 0) {
+ Thread.sleep(reduceSleepDuration);
+ }
+ }
+ catch (InterruptedException ex) {
+ throw (IOException)new IOException(
+ "Interrupted while sleeping").initCause(ex);
+ }
+ count++;
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(), new MRRSleepJob(), args);
+ System.exit(res);
+ }
+
+ private Credentials credentials = new Credentials();
+
+ public DAG createDAG(FileSystem remoteFs, Configuration conf, Path remoteStagingDir,
+ int numMapper, int numReducer, int iReduceStagesCount,
+ int numIReducer, long mapSleepTime, int mapSleepCount,
+ long reduceSleepTime, int reduceSleepCount,
+ long iReduceSleepTime, int iReduceSleepCount, boolean writeSplitsToDFS,
+ boolean generateSplitsInAM)
+ throws IOException, YarnException {
+
+
+ Configuration mapStageConf = new JobConf(conf);
+ mapStageConf.setInt(MRJobConfig.NUM_MAPS, numMapper);
+ mapStageConf.setLong(MAP_SLEEP_TIME, mapSleepTime);
+ mapStageConf.setLong(REDUCE_SLEEP_TIME, reduceSleepTime);
+ mapStageConf.setLong(IREDUCE_SLEEP_TIME, iReduceSleepTime);
+ mapStageConf.setInt(MAP_SLEEP_COUNT, mapSleepCount);
+ mapStageConf.setInt(REDUCE_SLEEP_COUNT, reduceSleepCount);
+ mapStageConf.setInt(IREDUCE_SLEEP_COUNT, iReduceSleepCount);
+ mapStageConf.setInt(IREDUCE_STAGES_COUNT, iReduceStagesCount);
+ mapStageConf.setInt(IREDUCE_TASKS_COUNT, numIReducer);
+ mapStageConf.set(MRJobConfig.MAP_CLASS_ATTR, SleepMapper.class.getName());
+ mapStageConf.set(MRJobConfig.INPUT_FORMAT_CLASS_ATTR,
+ SleepInputFormat.class.getName());
+ if (numIReducer == 0 && numReducer == 0) {
+ mapStageConf.set(MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR,
+ NullOutputFormat.class.getName());
+ }
+
+ MRHelpers.translateMRConfToTez(mapStageConf);
+
+ Configuration[] intermediateReduceStageConfs = null;
+ if (iReduceStagesCount > 0
+ && numIReducer > 0) {
+ intermediateReduceStageConfs = new JobConf[iReduceStagesCount];
+ for (int i = 1; i <= iReduceStagesCount; ++i) {
+ JobConf iReduceStageConf = new JobConf(conf);
+ iReduceStageConf.setLong(MRRSleepJob.REDUCE_SLEEP_TIME, iReduceSleepTime);
+ iReduceStageConf.setInt(MRRSleepJob.REDUCE_SLEEP_COUNT, iReduceSleepCount);
+ iReduceStageConf.setInt(MRJobConfig.NUM_REDUCES, numIReducer);
+ iReduceStageConf
+ .set(MRJobConfig.REDUCE_CLASS_ATTR, ISleepReducer.class.getName());
+ iReduceStageConf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS,
+ IntWritable.class.getName());
+ iReduceStageConf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS,
+ IntWritable.class.getName());
+ iReduceStageConf.set(MRJobConfig.PARTITIONER_CLASS_ATTR,
+ MRRSleepJobPartitioner.class.getName());
+
+
+ MRHelpers.translateMRConfToTez(iReduceStageConf);
+ intermediateReduceStageConfs[i-1] = iReduceStageConf;
+ }
+ }
+
+ Configuration finalReduceConf = null;
+ if (numReducer > 0) {
+ finalReduceConf = new JobConf(conf);
+ finalReduceConf.setLong(MRRSleepJob.REDUCE_SLEEP_TIME, reduceSleepTime);
+ finalReduceConf.setInt(MRRSleepJob.REDUCE_SLEEP_COUNT, reduceSleepCount);
+ finalReduceConf.setInt(MRJobConfig.NUM_REDUCES, numReducer);
+ finalReduceConf.set(MRJobConfig.REDUCE_CLASS_ATTR, SleepReducer.class.getName());
+ finalReduceConf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS,
+ IntWritable.class.getName());
+ finalReduceConf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS,
+ IntWritable.class.getName());
+ finalReduceConf.set(MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR,
+ NullOutputFormat.class.getName());
+
+ MRHelpers.translateMRConfToTez(finalReduceConf);
+ }
+
+ MRHelpers.configureMRApiUsage(mapStageConf);
+ if (iReduceStagesCount > 0
+ && numIReducer > 0) {
+ for (int i = 0; i < iReduceStagesCount; ++i) {
+ MRHelpers.configureMRApiUsage(intermediateReduceStageConfs[i]);
+ }
+ }
+ if (numReducer > 0) {
+ MRHelpers.configureMRApiUsage(finalReduceConf);
+ }
+
+ DataSourceDescriptor dataSource = null;
+ if (!generateSplitsInAM && writeSplitsToDFS) {
+
+ LOG.info("Writing splits to DFS");
+ dataSource = MRInputHelpers
+ .configureMRInputWithLegacySplitGeneration(mapStageConf, remoteStagingDir, true);
+ } else {
+ dataSource = MRInputLegacy.createConfigurer(mapStageConf, SleepInputFormat.class)
+ .generateSplitsInAM(generateSplitsInAM).create();
+ }
+
+ DAG dag = new DAG("MRRSleepJob");
+ String jarPath = ClassUtil.findContainingJar(getClass());
+ if (jarPath == null) {
+ throw new TezUncheckedException("Could not find any jar containing"
+ + " MRRSleepJob.class in the classpath");
+ }
+ Path remoteJarPath = remoteFs.makeQualified(
+ new Path(remoteStagingDir, "dag_job.jar"));
+ remoteFs.copyFromLocalFile(new Path(jarPath), remoteJarPath);
+ FileStatus jarFileStatus = remoteFs.getFileStatus(remoteJarPath);
+
+ TokenCache.obtainTokensForNamenodes(this.credentials, new Path[] { remoteJarPath },
+ mapStageConf);
+
+ Map<String, LocalResource> commonLocalResources =
+ new HashMap<String, LocalResource>();
+ LocalResource dagJarLocalRsrc = LocalResource.newInstance(
+ ConverterUtils.getYarnUrlFromPath(remoteJarPath),
+ LocalResourceType.FILE,
+ LocalResourceVisibility.APPLICATION,
+ jarFileStatus.getLen(),
+ jarFileStatus.getModificationTime());
+ commonLocalResources.put("dag_job.jar", dagJarLocalRsrc);
+
+ List<Vertex> vertices = new ArrayList<Vertex>();
+
+
+ UserPayload mapUserPayload = TezUtils.createUserPayloadFromConf(mapStageConf);
+ int numTasks = generateSplitsInAM ? -1 : numMapper;
+
+ Vertex mapVertex = new Vertex("map", new ProcessorDescriptor(
+ MapProcessor.class.getName()).setUserPayload(mapUserPayload), numTasks)
+ .setTaskLocalFiles(commonLocalResources);
+ mapVertex.addDataSource("MRInput", dataSource);
+ vertices.add(mapVertex);
+
+ if (iReduceStagesCount > 0
+ && numIReducer > 0) {
+ for (int i = 0; i < iReduceStagesCount; ++i) {
+ Configuration iconf =
+ intermediateReduceStageConfs[i];
+ UserPayload iReduceUserPayload = TezUtils.createUserPayloadFromConf(iconf);
+ Vertex ivertex = new Vertex("ireduce" + (i+1),
+ new ProcessorDescriptor(ReduceProcessor.class.getName()).
+ setUserPayload(iReduceUserPayload), numIReducer);
+ ivertex.setTaskLocalFiles(commonLocalResources);
+ vertices.add(ivertex);
+ }
+ }
+
+ Vertex finalReduceVertex = null;
+ if (numReducer > 0) {
+ UserPayload reducePayload = TezUtils.createUserPayloadFromConf(finalReduceConf);
+ finalReduceVertex = new Vertex("reduce", new ProcessorDescriptor(
+ ReduceProcessor.class.getName()).setUserPayload(reducePayload), numReducer);
+ finalReduceVertex.setTaskLocalFiles(commonLocalResources);
+ finalReduceVertex.addDataSink("MROutput", MROutputLegacy.createConfigurer(finalReduceConf,
+ NullOutputFormat.class).create());
+ vertices.add(finalReduceVertex);
+ } else {
+ // Map only job
+ mapVertex.addDataSink("MROutput",
+ MROutputLegacy.createConfigurer(mapStageConf, NullOutputFormat.class).create());
+ }
+
+
+ Map<String, String> partitionerConf = Maps.newHashMap();
+ partitionerConf.put(MRJobConfig.PARTITIONER_CLASS_ATTR, MRRSleepJobPartitioner.class.getName());
+ OrderedPartitionedKVEdgeConfigurer edgeConf = OrderedPartitionedKVEdgeConfigurer
+ .newBuilder(IntWritable.class.getName(), IntWritable.class.getName(),
+ HashPartitioner.class.getName(), partitionerConf).configureInput().useLegacyInput()
+ .done().build();
+
+ for (int i = 0; i < vertices.size(); ++i) {
+ dag.addVertex(vertices.get(i));
+ if (i != 0) {
+ dag.addEdge(
+ new Edge(vertices.get(i - 1), vertices.get(i), edgeConf.createDefaultEdgeProperty()));
+ }
+ }
+
+ return dag;
+ }
+
+ @VisibleForTesting
+ public Job createJob(int numMapper, int numReducer, int iReduceStagesCount,
+ int numIReducer, long mapSleepTime, int mapSleepCount,
+ long reduceSleepTime, int reduceSleepCount,
+ long iReduceSleepTime, int iReduceSleepCount)
+ throws IOException {
+ Configuration conf = getConf();
+ conf.setLong(MAP_SLEEP_TIME, mapSleepTime);
+ conf.setLong(REDUCE_SLEEP_TIME, reduceSleepTime);
+ conf.setLong(IREDUCE_SLEEP_TIME, iReduceSleepTime);
+ conf.setInt(MAP_SLEEP_COUNT, mapSleepCount);
+ conf.setInt(REDUCE_SLEEP_COUNT, reduceSleepCount);
+ conf.setInt(IREDUCE_SLEEP_COUNT, iReduceSleepCount);
+ conf.setInt(MRJobConfig.NUM_MAPS, numMapper);
+ conf.setInt(IREDUCE_STAGES_COUNT, iReduceStagesCount);
+ conf.setInt(IREDUCE_TASKS_COUNT, numIReducer);
+
+ // Configure intermediate reduces
+ conf.setInt(
+ org.apache.tez.mapreduce.hadoop.MRJobConfig.MRR_INTERMEDIATE_STAGES,
+ iReduceStagesCount);
+ LOG.info("Running MRR with " + iReduceStagesCount + " IR stages");
+
+ for (int i = 1; i <= iReduceStagesCount; ++i) {
+ // Set reducer class for intermediate reduce
+ conf.setClass(
+ MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,
+ "mapreduce.job.reduce.class"), ISleepReducer.class, Reducer.class);
+ // Set reducer output key class
+ conf.setClass(
+ MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,
+ "mapreduce.map.output.key.class"), IntWritable.class, Object.class);
+ // Set reducer output value class
+ conf.setClass(
+ MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,
+ "mapreduce.map.output.value.class"), IntWritable.class, Object.class);
+ conf.setInt(
+ MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,
+ "mapreduce.job.reduces"), numIReducer);
+ }
+
+ Job job = Job.getInstance(conf, "sleep");
+ job.setNumReduceTasks(numReducer);
+ job.setJarByClass(MRRSleepJob.class);
+ job.setNumReduceTasks(numReducer);
+ job.setMapperClass(SleepMapper.class);
+ job.setMapOutputKeyClass(IntWritable.class);
+ job.setMapOutputValueClass(IntWritable.class);
+ job.setReducerClass(SleepReducer.class);
+ job.setOutputFormatClass(NullOutputFormat.class);
+ job.setInputFormatClass(SleepInputFormat.class);
+ job.setPartitionerClass(MRRSleepJobPartitioner.class);
+ job.setSpeculativeExecution(false);
+ job.setJobName("Sleep job");
+
+ FileInputFormat.addInputPath(job, new Path("ignored"));
+ return job;
+ }
+
+ public int run(String[] args) throws Exception {
+
+ if(args.length < 1) {
+ System.err.println("MRRSleepJob [-m numMapper] [-r numReducer]" +
+ " [-ir numIntermediateReducer]" +
+ " [-irs numIntermediateReducerStages]" +
+ " [-mt mapSleepTime (msec)] [-rt reduceSleepTime (msec)]" +
+ " [-irt intermediateReduceSleepTime]" +
+ " [-recordt recordSleepTime (msec)]" +
+ " [-generateSplitsInAM (false)/true]" +
+ " [-writeSplitsToDfs (false)/true]");
+ ToolRunner.printGenericCommandUsage(System.err);
+ return 2;
+ }
+
+ int numMapper = 1, numReducer = 1, numIReducer = 1;
+ long mapSleepTime = 100, reduceSleepTime = 100, recSleepTime = 100,
+ iReduceSleepTime=1;
+ int mapSleepCount = 1, reduceSleepCount = 1, iReduceSleepCount = 1;
+ int iReduceStagesCount = 1;
+ boolean writeSplitsToDfs = false;
+ boolean generateSplitsInAM = false;
+ boolean splitsOptionFound = false;
+
+ for(int i=0; i < args.length; i++ ) {
+ if(args[i].equals("-m")) {
+ numMapper = Integer.parseInt(args[++i]);
+ }
+ else if(args[i].equals("-r")) {
+ numReducer = Integer.parseInt(args[++i]);
+ }
+ else if(args[i].equals("-ir")) {
+ numIReducer = Integer.parseInt(args[++i]);
+ }
+ else if(args[i].equals("-mt")) {
+ mapSleepTime = Long.parseLong(args[++i]);
+ }
+ else if(args[i].equals("-rt")) {
+ reduceSleepTime = Long.parseLong(args[++i]);
+ }
+ else if(args[i].equals("-irt")) {
+ iReduceSleepTime = Long.parseLong(args[++i]);
+ }
+ else if(args[i].equals("-irs")) {
+ iReduceStagesCount = Integer.parseInt(args[++i]);
+ }
+ else if (args[i].equals("-recordt")) {
+ recSleepTime = Long.parseLong(args[++i]);
+ }
+ else if (args[i].equals("-generateSplitsInAM")) {
+ if (splitsOptionFound) {
+ throw new RuntimeException("Cannot use both -generateSplitsInAm and -writeSplitsToDfs together");
+ }
+ splitsOptionFound = true;
+ generateSplitsInAM = Boolean.parseBoolean(args[++i]);
+
+ }
+ else if (args[i].equals("-writeSplitsToDfs")) {
+ if (splitsOptionFound) {
+ throw new RuntimeException("Cannot use both -generateSplitsInAm and -writeSplitsToDfs together");
+ }
+ splitsOptionFound = true;
+ writeSplitsToDfs = Boolean.parseBoolean(args[++i]);
+ }
+ }
+
+ if (numIReducer > 0 && numReducer <= 0) {
+ throw new RuntimeException("Cannot have intermediate reduces without"
+ + " a final reduce");
+ }
+
+ // sleep for *SleepTime duration in Task by recSleepTime per record
+ mapSleepCount = (int)Math.ceil(mapSleepTime / ((double)recSleepTime));
+ reduceSleepCount = (int)Math.ceil(reduceSleepTime / ((double)recSleepTime));
+ iReduceSleepCount = (int)Math.ceil(iReduceSleepTime / ((double)recSleepTime));
+
+ TezConfiguration conf = new TezConfiguration(getConf());
+ FileSystem remoteFs = FileSystem.get(conf);
+
+ conf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
+ conf.get(
+ TezConfiguration.TEZ_AM_STAGING_DIR,
+ TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT));
+
+ Path remoteStagingDir =
+ remoteFs.makeQualified(new Path(conf.get(
+ TezConfiguration.TEZ_AM_STAGING_DIR,
+ TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT),
+ Long.toString(System.currentTimeMillis())));
+ TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);
+
+ DAG dag = createDAG(remoteFs, conf, remoteStagingDir,
+ numMapper, numReducer, iReduceStagesCount, numIReducer,
+ mapSleepTime, mapSleepCount, reduceSleepTime, reduceSleepCount,
+ iReduceSleepTime, iReduceSleepCount, writeSplitsToDfs, generateSplitsInAM);
+
+ TezClient tezSession = new TezClient("MRRSleep", conf, false, null, credentials);
+ tezSession.start();
+ DAGClient dagClient = tezSession.submitDAG(dag);
+
+ while (true) {
+ DAGStatus status = dagClient.getDAGStatus(null);
+ LOG.info("DAG Status: " + status);
+ if (status.isCompleted()) {
+ break;
+ }
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ // do nothing
+ }
+ }
+ tezSession.stop();
+
+ return dagClient.getDAGStatus(null).getState().equals(DAGStatus.State.SUCCEEDED) ? 0 : 1;
+ }
+
+}
[07/10] TEZ-1055. Rename tez-mapreduce-examples to tez-examples
(Hitesh Shah via bikas)
Posted by bi...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/RandomWriter.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/RandomWriter.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/RandomWriter.java
deleted file mode 100644
index c9b51c7..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/RandomWriter.java
+++ /dev/null
@@ -1,298 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.mapred.ClusterStatus;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-/**
- * This program uses map/reduce to just run a distributed job where there is
- * no interaction between the tasks and each task write a large unsorted
- * random binary sequence file of BytesWritable.
- * In order for this program to generate data for terasort with 10-byte keys
- * and 90-byte values, have the following config:
- * <xmp>
- * <?xml version="1.0"?>
- * <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
- * <configuration>
- * <property>
- * <name>mapreduce.randomwriter.minkey</name>
- * <value>10</value>
- * </property>
- * <property>
- * <name>mapreduce.randomwriter.maxkey</name>
- * <value>10</value>
- * </property>
- * <property>
- * <name>mapreduce.randomwriter.minvalue</name>
- * <value>90</value>
- * </property>
- * <property>
- * <name>mapreduce.randomwriter.maxvalue</name>
- * <value>90</value>
- * </property>
- * <property>
- * <name>mapreduce.randomwriter.totalbytes</name>
- * <value>1099511627776</value>
- * </property>
- * </configuration></xmp>
- *
- * Equivalently, {@link RandomWriter} also supports all the above options
- * and ones supported by {@link GenericOptionsParser} via the command-line.
- */
-public class RandomWriter extends Configured implements Tool {
- public static final String TOTAL_BYTES = "mapreduce.randomwriter.totalbytes";
- public static final String BYTES_PER_MAP =
- "mapreduce.randomwriter.bytespermap";
- public static final String MAPS_PER_HOST =
- "mapreduce.randomwriter.mapsperhost";
- public static final String MAX_VALUE = "mapreduce.randomwriter.maxvalue";
- public static final String MIN_VALUE = "mapreduce.randomwriter.minvalue";
- public static final String MIN_KEY = "mapreduce.randomwriter.minkey";
- public static final String MAX_KEY = "mapreduce.randomwriter.maxkey";
-
- /**
- * User counters
- */
- static enum Counters { RECORDS_WRITTEN, BYTES_WRITTEN }
-
- /**
- * A custom input format that creates virtual inputs of a single string
- * for each map.
- */
- static class RandomInputFormat extends InputFormat<Text, Text> {
-
- /**
- * Generate the requested number of file splits, with the filename
- * set to the filename of the output file.
- */
- public List<InputSplit> getSplits(JobContext job) throws IOException {
- List<InputSplit> result = new ArrayList<InputSplit>();
- Path outDir = FileOutputFormat.getOutputPath(job);
- int numSplits =
- job.getConfiguration().getInt(MRJobConfig.NUM_MAPS, 1);
- for(int i=0; i < numSplits; ++i) {
- result.add(new FileSplit(new Path(outDir, "dummy-split-" + i), 0, 1,
- (String[])null));
- }
- return result;
- }
-
- /**
- * Return a single record (filename, "") where the filename is taken from
- * the file split.
- */
- static class RandomRecordReader extends RecordReader<Text, Text> {
- Path name;
- Text key = null;
- Text value = new Text();
- public RandomRecordReader(Path p) {
- name = p;
- }
-
- public void initialize(InputSplit split,
- TaskAttemptContext context)
- throws IOException, InterruptedException {
-
- }
-
- public boolean nextKeyValue() {
- if (name != null) {
- key = new Text();
- key.set(name.getName());
- name = null;
- return true;
- }
- return false;
- }
-
- public Text getCurrentKey() {
- return key;
- }
-
- public Text getCurrentValue() {
- return value;
- }
-
- public void close() {}
-
- public float getProgress() {
- return 0.0f;
- }
- }
-
- public RecordReader<Text, Text> createRecordReader(InputSplit split,
- TaskAttemptContext context) throws IOException, InterruptedException {
- return new RandomRecordReader(((FileSplit) split).getPath());
- }
- }
-
- static class RandomMapper extends Mapper<WritableComparable, Writable,
- BytesWritable, BytesWritable> {
-
- private long numBytesToWrite;
- private int minKeySize;
- private int keySizeRange;
- private int minValueSize;
- private int valueSizeRange;
- private Random random = new Random();
- private BytesWritable randomKey = new BytesWritable();
- private BytesWritable randomValue = new BytesWritable();
-
- private void randomizeBytes(byte[] data, int offset, int length) {
- for(int i=offset + length - 1; i >= offset; --i) {
- data[i] = (byte) random.nextInt(256);
- }
- }
-
- /**
- * Given an output filename, write a bunch of random records to it.
- */
- public void map(WritableComparable key,
- Writable value,
- Context context) throws IOException,InterruptedException {
- int itemCount = 0;
- while (numBytesToWrite > 0) {
- int keyLength = minKeySize +
- (keySizeRange != 0 ? random.nextInt(keySizeRange) : 0);
- randomKey.setSize(keyLength);
- randomizeBytes(randomKey.getBytes(), 0, randomKey.getLength());
- int valueLength = minValueSize +
- (valueSizeRange != 0 ? random.nextInt(valueSizeRange) : 0);
- randomValue.setSize(valueLength);
- randomizeBytes(randomValue.getBytes(), 0, randomValue.getLength());
- context.write(randomKey, randomValue);
- numBytesToWrite -= keyLength + valueLength;
- context.getCounter(Counters.BYTES_WRITTEN).increment(keyLength + valueLength);
- context.getCounter(Counters.RECORDS_WRITTEN).increment(1);
- if (++itemCount % 200 == 0) {
- context.setStatus("wrote record " + itemCount + ". " +
- numBytesToWrite + " bytes left.");
- }
- }
- context.setStatus("done with " + itemCount + " records.");
- }
-
- /**
- * Save the values out of the configuaration that we need to write
- * the data.
- */
- @Override
- public void setup(Context context) {
- Configuration conf = context.getConfiguration();
- numBytesToWrite = conf.getLong(BYTES_PER_MAP,
- 1*1024*1024*1024);
- minKeySize = conf.getInt(MIN_KEY, 10);
- keySizeRange =
- conf.getInt(MAX_KEY, 1000) - minKeySize;
- minValueSize = conf.getInt(MIN_VALUE, 0);
- valueSizeRange =
- conf.getInt(MAX_VALUE, 20000) - minValueSize;
- }
- }
-
- /**
- * This is the main routine for launching a distributed random write job.
- * It runs 10 maps/node and each node writes 1 gig of data to a DFS file.
- * The reduce doesn't do anything.
- *
- * @throws IOException
- */
- public int run(String[] args) throws Exception {
- if (args.length == 0) {
- System.out.println("Usage: writer <out-dir>");
- ToolRunner.printGenericCommandUsage(System.out);
- return 2;
- }
-
- Path outDir = new Path(args[0]);
- Configuration conf = getConf();
- JobClient client = new JobClient(conf);
- ClusterStatus cluster = client.getClusterStatus();
- int numMapsPerHost = conf.getInt(MAPS_PER_HOST, 10);
- long numBytesToWritePerMap = conf.getLong(BYTES_PER_MAP,
- 1*1024*1024*1024);
- if (numBytesToWritePerMap == 0) {
- System.err.println("Cannot have" + BYTES_PER_MAP + " set to 0");
- return -2;
- }
- long totalBytesToWrite = conf.getLong(TOTAL_BYTES,
- numMapsPerHost*numBytesToWritePerMap*cluster.getTaskTrackers());
- int numMaps = (int) (totalBytesToWrite / numBytesToWritePerMap);
- if (numMaps == 0 && totalBytesToWrite > 0) {
- numMaps = 1;
- conf.setLong(BYTES_PER_MAP, totalBytesToWrite);
- }
- conf.setInt(MRJobConfig.NUM_MAPS, numMaps);
-
- Job job = new Job(conf);
-
- job.setJarByClass(RandomWriter.class);
- job.setJobName("random-writer");
- FileOutputFormat.setOutputPath(job, outDir);
- job.setOutputKeyClass(BytesWritable.class);
- job.setOutputValueClass(BytesWritable.class);
- job.setInputFormatClass(RandomInputFormat.class);
- job.setMapperClass(RandomMapper.class);
- job.setReducerClass(Reducer.class);
- job.setOutputFormatClass(SequenceFileOutputFormat.class);
-
- System.out.println("Running " + numMaps + " maps.");
-
- // reducer NONE
- job.setNumReduceTasks(0);
-
- Date startTime = new Date();
- System.out.println("Job started: " + startTime);
- int ret = job.waitForCompletion(true) ? 0 : 1;
- Date endTime = new Date();
- System.out.println("Job ended: " + endTime);
- System.out.println("The job took " +
- (endTime.getTime() - startTime.getTime()) /1000 +
- " seconds.");
-
- return ret;
- }
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new RandomWriter(), args);
- System.exit(res);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/SecondarySort.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/SecondarySort.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/SecondarySort.java
deleted file mode 100644
index cdae905..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/SecondarySort.java
+++ /dev/null
@@ -1,248 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.StringTokenizer;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.RawComparator;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.io.WritableComparator;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Partitioner;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-/**
- * This is an example Hadoop Map/Reduce application.
- * It reads the text input files that must contain two integers per a line.
- * The output is sorted by the first and second number and grouped on the
- * first number.
- *
- * To run: bin/hadoop jar build/hadoop-examples.jar secondarysort
- * <i>in-dir</i> <i>out-dir</i>
- */
-public class SecondarySort extends Configured implements Tool {
-
- /**
- * Define a pair of integers that are writable.
- * They are serialized in a byte comparable format.
- */
- public static class IntPair
- implements WritableComparable<IntPair> {
- private int first = 0;
- private int second = 0;
-
- /**
- * Set the left and right values.
- */
- public void set(int left, int right) {
- first = left;
- second = right;
- }
- public int getFirst() {
- return first;
- }
- public int getSecond() {
- return second;
- }
- /**
- * Read the two integers.
- * Encoded as: MIN_VALUE -> 0, 0 -> -MIN_VALUE, MAX_VALUE-> -1
- */
- @Override
- public void readFields(DataInput in) throws IOException {
- first = in.readInt() + Integer.MIN_VALUE;
- second = in.readInt() + Integer.MIN_VALUE;
- }
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeInt(first - Integer.MIN_VALUE);
- out.writeInt(second - Integer.MIN_VALUE);
- }
- @Override
- public int hashCode() {
- return first * 157 + second;
- }
- @Override
- public boolean equals(Object right) {
- if (right instanceof IntPair) {
- IntPair r = (IntPair) right;
- return r.first == first && r.second == second;
- } else {
- return false;
- }
- }
- /** A Comparator that compares serialized IntPair. */
- public static class Comparator extends WritableComparator {
- public Comparator() {
- super(IntPair.class);
- }
-
- public int compare(byte[] b1, int s1, int l1,
- byte[] b2, int s2, int l2) {
- return compareBytes(b1, s1, l1, b2, s2, l2);
- }
- }
-
- static { // register this comparator
- WritableComparator.define(IntPair.class, new Comparator());
- }
-
- @Override
- public int compareTo(IntPair o) {
- if (first != o.first) {
- return first < o.first ? -1 : 1;
- } else if (second != o.second) {
- return second < o.second ? -1 : 1;
- } else {
- return 0;
- }
- }
- }
-
- /**
- * Partition based on the first part of the pair.
- */
- public static class FirstPartitioner extends Partitioner<IntPair,IntWritable>{
- @Override
- public int getPartition(IntPair key, IntWritable value,
- int numPartitions) {
- return Math.abs(key.getFirst() * 127) % numPartitions;
- }
- }
-
- /**
- * Compare only the first part of the pair, so that reduce is called once
- * for each value of the first part.
- */
- public static class FirstGroupingComparator
- implements RawComparator<IntPair> {
- @Override
- public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
- return WritableComparator.compareBytes(b1, s1, Integer.SIZE/8,
- b2, s2, Integer.SIZE/8);
- }
-
- @Override
- public int compare(IntPair o1, IntPair o2) {
- int l = o1.getFirst();
- int r = o2.getFirst();
- return l == r ? 0 : (l < r ? -1 : 1);
- }
- }
-
- /**
- * Read two integers from each line and generate a key, value pair
- * as ((left, right), right).
- */
- public static class MapClass
- extends Mapper<LongWritable, Text, IntPair, IntWritable> {
-
- private final IntPair key = new IntPair();
- private final IntWritable value = new IntWritable();
-
- @Override
- public void map(LongWritable inKey, Text inValue,
- Context context) throws IOException, InterruptedException {
- StringTokenizer itr = new StringTokenizer(inValue.toString());
- int left = 0;
- int right = 0;
- if (itr.hasMoreTokens()) {
- left = Integer.parseInt(itr.nextToken());
- if (itr.hasMoreTokens()) {
- right = Integer.parseInt(itr.nextToken());
- }
- key.set(left, right);
- value.set(right);
- context.write(key, value);
- }
- }
- }
-
- /**
- * A reducer class that just emits the sum of the input values.
- */
- public static class Reduce
- extends Reducer<IntPair, IntWritable, Text, IntWritable> {
- private static final Text SEPARATOR =
- new Text("------------------------------------------------");
- private final Text first = new Text();
-
- @Override
- public void reduce(IntPair key, Iterable<IntWritable> values,
- Context context
- ) throws IOException, InterruptedException {
- context.write(SEPARATOR, null);
- first.set(Integer.toString(key.getFirst()));
- for(IntWritable value: values) {
- context.write(first, value);
- }
- }
- }
-
- @Override
- public int run(String[] args) throws Exception {
- Configuration conf = getConf();
- String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
- if (otherArgs.length != 2) {
- System.err.println("Usage: secondarysort <in> <out>");
- ToolRunner.printGenericCommandUsage(System.out);
- System.exit(2);
- }
- Job job = new Job(conf, "secondary sort");
- job.setJarByClass(SecondarySort.class);
- job.setMapperClass(MapClass.class);
- job.setReducerClass(Reduce.class);
-
- // group and partition by the first int in the pair
- job.setPartitionerClass(FirstPartitioner.class);
- job.setGroupingComparatorClass(FirstGroupingComparator.class);
-
- // the map output is IntPair, IntWritable
- job.setMapOutputKeyClass(IntPair.class);
- job.setMapOutputValueClass(IntWritable.class);
-
- // the reduce output is Text, IntWritable
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
-
- FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
- FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
- return job.waitForCompletion(true) ? 0 : 1;
- }
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new SecondarySort(), args);
- System.exit(res);
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/Sort.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/Sort.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/Sort.java
deleted file mode 100644
index 4eb5484..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/Sort.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples;
-
-import java.net.URI;
-import java.util.*;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.mapreduce.filecache.DistributedCache;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.mapred.ClusterStatus;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.partition.InputSampler;
-import org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-/**
- * This is the trivial map/reduce program that does absolutely nothing
- * other than use the framework to fragment and sort the input values.
- *
- * To run: bin/hadoop jar build/hadoop-examples.jar sort
- * [-r <i>reduces</i>]
- * [-inFormat <i>input format class</i>]
- * [-outFormat <i>output format class</i>]
- * [-outKey <i>output key class</i>]
- * [-outValue <i>output value class</i>]
- * [-totalOrder <i>pcnt</i> <i>num samples</i> <i>max splits</i>]
- * <i>in-dir</i> <i>out-dir</i>
- */
-public class Sort<K,V> extends Configured implements Tool {
- public static final String REDUCES_PER_HOST =
- "mapreduce.sort.reducesperhost";
- private Job job = null;
-
- static int printUsage() {
- System.out.println("sort [-r <reduces>] " +
- "[-inFormat <input format class>] " +
- "[-outFormat <output format class>] " +
- "[-outKey <output key class>] " +
- "[-outValue <output value class>] " +
- "[-totalOrder <pcnt> <num samples> <max splits>] " +
- "<input> <output>");
- ToolRunner.printGenericCommandUsage(System.out);
- return 2;
- }
-
- /**
- * The main driver for sort program.
- * Invoke this method to submit the map/reduce job.
- * @throws IOException When there is communication problems with the
- * job tracker.
- */
- public int run(String[] args) throws Exception {
-
- Configuration conf = getConf();
- JobClient client = new JobClient(conf);
- ClusterStatus cluster = client.getClusterStatus();
- int num_reduces = (int) (cluster.getMaxReduceTasks() * 0.9);
- String sort_reduces = conf.get(REDUCES_PER_HOST);
- if (sort_reduces != null) {
- num_reduces = cluster.getTaskTrackers() *
- Integer.parseInt(sort_reduces);
- }
- Class<? extends InputFormat> inputFormatClass =
- SequenceFileInputFormat.class;
- Class<? extends OutputFormat> outputFormatClass =
- SequenceFileOutputFormat.class;
- Class<? extends WritableComparable> outputKeyClass = BytesWritable.class;
- Class<? extends Writable> outputValueClass = BytesWritable.class;
- List<String> otherArgs = new ArrayList<String>();
- InputSampler.Sampler<K,V> sampler = null;
- for(int i=0; i < args.length; ++i) {
- try {
- if ("-r".equals(args[i])) {
- num_reduces = Integer.parseInt(args[++i]);
- } else if ("-inFormat".equals(args[i])) {
- inputFormatClass =
- Class.forName(args[++i]).asSubclass(InputFormat.class);
- } else if ("-outFormat".equals(args[i])) {
- outputFormatClass =
- Class.forName(args[++i]).asSubclass(OutputFormat.class);
- } else if ("-outKey".equals(args[i])) {
- outputKeyClass =
- Class.forName(args[++i]).asSubclass(WritableComparable.class);
- } else if ("-outValue".equals(args[i])) {
- outputValueClass =
- Class.forName(args[++i]).asSubclass(Writable.class);
- } else if ("-totalOrder".equals(args[i])) {
- double pcnt = Double.parseDouble(args[++i]);
- int numSamples = Integer.parseInt(args[++i]);
- int maxSplits = Integer.parseInt(args[++i]);
- if (0 >= maxSplits) maxSplits = Integer.MAX_VALUE;
- sampler =
- new InputSampler.RandomSampler<K,V>(pcnt, numSamples, maxSplits);
- } else {
- otherArgs.add(args[i]);
- }
- } catch (NumberFormatException except) {
- System.out.println("ERROR: Integer expected instead of " + args[i]);
- return printUsage();
- } catch (ArrayIndexOutOfBoundsException except) {
- System.out.println("ERROR: Required parameter missing from " +
- args[i-1]);
- return printUsage(); // exits
- }
- }
- // Set user-supplied (possibly default) job configs
- job = new Job(conf);
- job.setJobName("sorter");
- job.setJarByClass(Sort.class);
-
- job.setMapperClass(Mapper.class);
- job.setReducerClass(Reducer.class);
-
- job.setNumReduceTasks(num_reduces);
-
- job.setInputFormatClass(inputFormatClass);
- job.setOutputFormatClass(outputFormatClass);
-
- job.setOutputKeyClass(outputKeyClass);
- job.setOutputValueClass(outputValueClass);
-
- // Make sure there are exactly 2 parameters left.
- if (otherArgs.size() != 2) {
- System.out.println("ERROR: Wrong number of parameters: " +
- otherArgs.size() + " instead of 2.");
- return printUsage();
- }
- FileInputFormat.setInputPaths(job, otherArgs.get(0));
- FileOutputFormat.setOutputPath(job, new Path(otherArgs.get(1)));
-
- if (sampler != null) {
- System.out.println("Sampling input to effect total-order sort...");
- job.setPartitionerClass(TotalOrderPartitioner.class);
- Path inputDir = FileInputFormat.getInputPaths(job)[0];
- inputDir = inputDir.makeQualified(inputDir.getFileSystem(conf));
- Path partitionFile = new Path(inputDir, "_sortPartitioning");
- TotalOrderPartitioner.setPartitionFile(conf, partitionFile);
- InputSampler.<K,V>writePartitionFile(job, sampler);
- URI partitionUri = new URI(partitionFile.toString() +
- "#" + "_sortPartitioning");
- DistributedCache.addCacheFile(partitionUri, conf);
- }
-
- System.out.println("Running on " +
- cluster.getTaskTrackers() +
- " nodes to sort from " +
- FileInputFormat.getInputPaths(job)[0] + " into " +
- FileOutputFormat.getOutputPath(job) +
- " with " + num_reduces + " reduces.");
- Date startTime = new Date();
- System.out.println("Job started: " + startTime);
- int ret = job.waitForCompletion(true) ? 0 : 1;
- Date end_time = new Date();
- System.out.println("Job ended: " + end_time);
- System.out.println("The job took " +
- (end_time.getTime() - startTime.getTime()) /1000 + " seconds.");
- return ret;
- }
-
-
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new Sort(), args);
- System.exit(res);
- }
-
- /**
- * Get the last job that was run using this instance.
- * @return the results of the last job that was run
- */
- public Job getResult() {
- return job;
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
deleted file mode 100644
index 72c14fc..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
+++ /dev/null
@@ -1,479 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.StringTokenizer;
-import java.util.TreeMap;
-
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.FileAlreadyExistsException;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
-import org.apache.hadoop.mapreduce.security.TokenCache;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.tez.client.PreWarmVertex;
-import org.apache.tez.client.TezClientUtils;
-import org.apache.tez.client.TezClient;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.DataSourceDescriptor;
-import org.apache.tez.dag.api.Edge;
-import org.apache.tez.dag.api.ProcessorDescriptor;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.api.Vertex;
-import org.apache.tez.dag.api.client.DAGClient;
-import org.apache.tez.dag.api.client.DAGStatus;
-import org.apache.tez.dag.api.client.StatusGetOpts;
-import org.apache.tez.mapreduce.examples.helpers.SplitsInClientOptionParser;
-import org.apache.tez.mapreduce.hadoop.MRHelpers;
-import org.apache.tez.mapreduce.hadoop.MRInputHelpers;
-import org.apache.tez.mapreduce.hadoop.MRJobConfig;
-import org.apache.tez.mapreduce.input.MRInputLegacy;
-import org.apache.tez.mapreduce.output.MROutputLegacy;
-import org.apache.tez.mapreduce.processor.map.MapProcessor;
-import org.apache.tez.mapreduce.processor.reduce.ReduceProcessor;
-import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
-import org.apache.tez.runtime.library.conf.OrderedPartitionedKVEdgeConfigurer;
-import org.apache.tez.runtime.library.partitioner.HashPartitioner;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * An MRR job built on top of word count to return words sorted by
- * their frequency of occurrence.
- *
- * Use -DUSE_TEZ_SESSION=true to run jobs in a session mode.
- * If multiple input/outputs are provided, this job will process each pair
- * as a separate DAG in a sequential manner.
- * Use -DINTER_JOB_SLEEP_INTERVAL=<N> where N is the sleep interval in seconds
- * between the sequential DAGs.
- */
-public class TestOrderedWordCount extends Configured implements Tool {
-
- private static Log LOG = LogFactory.getLog(TestOrderedWordCount.class);
-
- public static class TokenizerMapper
- extends Mapper<Object, Text, Text, IntWritable>{
-
- private final static IntWritable one = new IntWritable(1);
- private Text word = new Text();
-
- public void map(Object key, Text value, Context context
- ) throws IOException, InterruptedException {
- StringTokenizer itr = new StringTokenizer(value.toString());
- while (itr.hasMoreTokens()) {
- word.set(itr.nextToken());
- context.write(word, one);
- }
- }
- }
-
- public static class IntSumReducer
- extends Reducer<Text,IntWritable,IntWritable, Text> {
- private IntWritable result = new IntWritable();
-
- public void reduce(Text key, Iterable<IntWritable> values,
- Context context
- ) throws IOException, InterruptedException {
- int sum = 0;
- for (IntWritable val : values) {
- sum += val.get();
- }
- result.set(sum);
- context.write(result, key);
- }
- }
-
- /**
- * Shuffle ensures ordering based on count of employees per department
- * hence the final reducer is a no-op and just emits the department name
- * with the employee count per department.
- */
- public static class MyOrderByNoOpReducer
- extends Reducer<IntWritable, Text, Text, IntWritable> {
-
- public void reduce(IntWritable key, Iterable<Text> values,
- Context context
- ) throws IOException, InterruptedException {
- for (Text word : values) {
- context.write(word, key);
- }
- }
- }
-
- private Credentials credentials = new Credentials();
-
- @VisibleForTesting
- public DAG createDAG(FileSystem fs, Configuration conf,
- Map<String, LocalResource> commonLocalResources, Path stagingDir,
- int dagIndex, String inputPath, String outputPath,
- boolean generateSplitsInClient) throws Exception {
-
- Configuration mapStageConf = new JobConf(conf);
- mapStageConf.set(MRJobConfig.MAP_CLASS_ATTR,
- TokenizerMapper.class.getName());
-
- MRHelpers.translateMRConfToTez(mapStageConf);
-
- Configuration iReduceStageConf = new JobConf(conf);
- // TODO replace with auto-reduce parallelism
- iReduceStageConf.setInt(MRJobConfig.NUM_REDUCES, 2);
- iReduceStageConf.set(MRJobConfig.REDUCE_CLASS_ATTR,
- IntSumReducer.class.getName());
- iReduceStageConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, Text.class.getName());
- iReduceStageConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS,
- IntWritable.class.getName());
- iReduceStageConf.setBoolean("mapred.mapper.new-api", true);
- MRHelpers.translateMRConfToTez(iReduceStageConf);
-
- Configuration finalReduceConf = new JobConf(conf);
- finalReduceConf.setInt(MRJobConfig.NUM_REDUCES, 1);
- finalReduceConf.set(MRJobConfig.REDUCE_CLASS_ATTR,
- MyOrderByNoOpReducer.class.getName());
- finalReduceConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, IntWritable.class.getName());
- finalReduceConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, Text.class.getName());
- MRHelpers.translateMRConfToTez(finalReduceConf);
-
- MRHelpers.configureMRApiUsage(mapStageConf);
- MRHelpers.configureMRApiUsage(iReduceStageConf);
- MRHelpers.configureMRApiUsage(finalReduceConf);
-
- List<Vertex> vertices = new ArrayList<Vertex>();
-
- ByteArrayOutputStream outputStream = new ByteArrayOutputStream(4096);
- mapStageConf.writeXml(outputStream);
- String mapStageHistoryText = new String(outputStream.toByteArray(), "UTF-8");
- DataSourceDescriptor dsd;
- if (generateSplitsInClient) {
- mapStageConf.set(MRJobConfig.INPUT_FORMAT_CLASS_ATTR,
- TextInputFormat.class.getName());
- mapStageConf.set(FileInputFormat.INPUT_DIR, inputPath);
- mapStageConf.setBoolean("mapred.mapper.new-api", true);
- dsd = MRInputHelpers.configureMRInputWithLegacySplitGeneration(mapStageConf, stagingDir, true);
- } else {
- dsd = MRInputLegacy.createConfigurer(mapStageConf, TextInputFormat.class, inputPath).create();
- }
-
- Vertex mapVertex = new Vertex("initialmap", new ProcessorDescriptor(
- MapProcessor.class.getName()).setUserPayload(
- TezUtils.createUserPayloadFromConf(mapStageConf))
- .setHistoryText(mapStageHistoryText)).setTaskLocalFiles(commonLocalResources);
- mapVertex.addDataSource("MRInput", dsd);
- vertices.add(mapVertex);
-
- ByteArrayOutputStream iROutputStream = new ByteArrayOutputStream(4096);
- iReduceStageConf.writeXml(iROutputStream);
- String iReduceStageHistoryText = new String(iROutputStream.toByteArray(), "UTF-8");
- Vertex ivertex = new Vertex("intermediate_reducer", new ProcessorDescriptor(
- ReduceProcessor.class.getName())
- .setUserPayload(TezUtils.createUserPayloadFromConf(iReduceStageConf))
- .setHistoryText(iReduceStageHistoryText), 2);
- ivertex.setTaskLocalFiles(commonLocalResources);
- vertices.add(ivertex);
-
- ByteArrayOutputStream finalReduceOutputStream = new ByteArrayOutputStream(4096);
- finalReduceConf.writeXml(finalReduceOutputStream);
- String finalReduceStageHistoryText = new String(finalReduceOutputStream.toByteArray(), "UTF-8");
- UserPayload finalReducePayload = TezUtils.createUserPayloadFromConf(finalReduceConf);
- Vertex finalReduceVertex = new Vertex("finalreduce",
- new ProcessorDescriptor(
- ReduceProcessor.class.getName())
- .setUserPayload(finalReducePayload)
- .setHistoryText(finalReduceStageHistoryText), 1);
- finalReduceVertex.setTaskLocalFiles(commonLocalResources);
- finalReduceVertex.addDataSink("MROutput",
- MROutputLegacy.createConfigurer(finalReduceConf, TextOutputFormat.class, outputPath)
- .create());
- vertices.add(finalReduceVertex);
-
- DAG dag = new DAG("OrderedWordCount" + dagIndex);
- for (int i = 0; i < vertices.size(); ++i) {
- dag.addVertex(vertices.get(i));
- }
-
- OrderedPartitionedKVEdgeConfigurer edgeConf1 = OrderedPartitionedKVEdgeConfigurer
- .newBuilder(Text.class.getName(), IntWritable.class.getName(),
- HashPartitioner.class.getName()).setFromConfiguration(conf)
- .configureInput().useLegacyInput().done().build();
- dag.addEdge(
- new Edge(dag.getVertex("initialmap"), dag.getVertex("intermediate_reducer"),
- edgeConf1.createDefaultEdgeProperty()));
-
- OrderedPartitionedKVEdgeConfigurer edgeConf2 = OrderedPartitionedKVEdgeConfigurer
- .newBuilder(IntWritable.class.getName(), Text.class.getName(),
- HashPartitioner.class.getName()).setFromConfiguration(conf)
- .configureInput().useLegacyInput().done().build();
- dag.addEdge(
- new Edge(dag.getVertex("intermediate_reducer"), dag.getVertex("finalreduce"),
- edgeConf2.createDefaultEdgeProperty()));
-
- return dag;
- }
-
- private static void printUsage() {
- String options = " [-generateSplitsInClient true/<false>]";
- System.err.println("Usage: testorderedwordcount <in> <out>" + options);
- System.err.println("Usage (In Session Mode):"
- + " testorderedwordcount <in1> <out1> ... <inN> <outN>" + options);
- ToolRunner.printGenericCommandUsage(System.err);
- }
-
-
- @Override
- public int run(String[] args) throws Exception {
- Configuration conf = getConf();
- String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
-
- boolean generateSplitsInClient;
-
- SplitsInClientOptionParser splitCmdLineParser = new SplitsInClientOptionParser();
- try {
- generateSplitsInClient = splitCmdLineParser.parse(otherArgs, false);
- otherArgs = splitCmdLineParser.getRemainingArgs();
- } catch (ParseException e1) {
- System.err.println("Invalid options");
- printUsage();
- return 2;
- }
-
- boolean useTezSession = conf.getBoolean("USE_TEZ_SESSION", true);
- long interJobSleepTimeout = conf.getInt("INTER_JOB_SLEEP_INTERVAL", 0)
- * 1000;
-
- boolean retainStagingDir = conf.getBoolean("RETAIN_STAGING_DIR", false);
-
- if (((otherArgs.length%2) != 0)
- || (!useTezSession && otherArgs.length != 2)) {
- printUsage();
- return 2;
- }
-
- List<String> inputPaths = new ArrayList<String>();
- List<String> outputPaths = new ArrayList<String>();
-
- for (int i = 0; i < otherArgs.length; i+=2) {
- inputPaths.add(otherArgs[i]);
- outputPaths.add(otherArgs[i+1]);
- }
-
- UserGroupInformation.setConfiguration(conf);
-
- TezConfiguration tezConf = new TezConfiguration(conf);
- TestOrderedWordCount instance = new TestOrderedWordCount();
-
- FileSystem fs = FileSystem.get(conf);
-
- String stagingDirStr = conf.get(TezConfiguration.TEZ_AM_STAGING_DIR,
- TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT) + Path.SEPARATOR +
- Long.toString(System.currentTimeMillis());
- Path stagingDir = new Path(stagingDirStr);
- FileSystem pathFs = stagingDir.getFileSystem(tezConf);
- pathFs.mkdirs(new Path(stagingDirStr));
-
- tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr);
- stagingDir = pathFs.makeQualified(new Path(stagingDirStr));
-
- TokenCache.obtainTokensForNamenodes(instance.credentials, new Path[] {stagingDir}, conf);
- TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
-
- // No need to add jar containing this class as assumed to be part of
- // the tez jars.
-
- // TEZ-674 Obtain tokens based on the Input / Output paths. For now assuming staging dir
- // is the same filesystem as the one used for Input/Output.
-
- if (useTezSession) {
- LOG.info("Creating Tez Session");
- tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
- } else {
- tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
- }
- TezClient tezSession = new TezClient("OrderedWordCountSession", tezConf,
- null, instance.credentials);
- tezSession.start();
-
- DAGStatus dagStatus = null;
- DAGClient dagClient = null;
- String[] vNames = { "initialmap", "intermediate_reducer",
- "finalreduce" };
-
- Set<StatusGetOpts> statusGetOpts = EnumSet.of(StatusGetOpts.GET_COUNTERS);
- try {
- for (int dagIndex = 1; dagIndex <= inputPaths.size(); ++dagIndex) {
- if (dagIndex != 1
- && interJobSleepTimeout > 0) {
- try {
- LOG.info("Sleeping between jobs, sleepInterval="
- + (interJobSleepTimeout/1000));
- Thread.sleep(interJobSleepTimeout);
- } catch (InterruptedException e) {
- LOG.info("Main thread interrupted. Breaking out of job loop");
- break;
- }
- }
-
- String inputPath = inputPaths.get(dagIndex-1);
- String outputPath = outputPaths.get(dagIndex-1);
-
- if (fs.exists(new Path(outputPath))) {
- throw new FileAlreadyExistsException("Output directory "
- + outputPath + " already exists");
- }
- LOG.info("Running OrderedWordCount DAG"
- + ", dagIndex=" + dagIndex
- + ", inputPath=" + inputPath
- + ", outputPath=" + outputPath);
-
- Map<String, LocalResource> localResources =
- new TreeMap<String, LocalResource>();
-
- DAG dag = instance.createDAG(fs, conf, localResources,
- stagingDir, dagIndex, inputPath, outputPath,
- generateSplitsInClient);
-
- boolean doPreWarm = dagIndex == 1 && useTezSession
- && conf.getBoolean("PRE_WARM_SESSION", true);
- int preWarmNumContainers = 0;
- if (doPreWarm) {
- preWarmNumContainers = conf.getInt("PRE_WARM_NUM_CONTAINERS", 0);
- if (preWarmNumContainers <= 0) {
- doPreWarm = false;
- }
- }
- if (doPreWarm) {
- LOG.info("Pre-warming Session");
- PreWarmVertex preWarmVertex = new PreWarmVertex("PreWarm", preWarmNumContainers, dag
- .getVertex("initialmap").getTaskResource());
- preWarmVertex.setTaskLocalFiles(dag.getVertex("initialmap").getTaskLocalFiles());
- preWarmVertex.setTaskEnvironment(dag.getVertex("initialmap").getTaskEnvironment());
- preWarmVertex.setTaskLaunchCmdOpts(dag.getVertex("initialmap").getTaskLaunchCmdOpts());
-
- tezSession.preWarm(preWarmVertex);
- }
-
- if (useTezSession) {
- LOG.info("Waiting for TezSession to get into ready state");
- waitForTezSessionReady(tezSession);
- LOG.info("Submitting DAG to Tez Session, dagIndex=" + dagIndex);
- dagClient = tezSession.submitDAG(dag);
- LOG.info("Submitted DAG to Tez Session, dagIndex=" + dagIndex);
- } else {
- LOG.info("Submitting DAG as a new Tez Application");
- dagClient = tezSession.submitDAG(dag);
- }
-
- while (true) {
- dagStatus = dagClient.getDAGStatus(statusGetOpts);
- if (dagStatus.getState() == DAGStatus.State.RUNNING ||
- dagStatus.getState() == DAGStatus.State.SUCCEEDED ||
- dagStatus.getState() == DAGStatus.State.FAILED ||
- dagStatus.getState() == DAGStatus.State.KILLED ||
- dagStatus.getState() == DAGStatus.State.ERROR) {
- break;
- }
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- // continue;
- }
- }
-
-
- while (dagStatus.getState() != DAGStatus.State.SUCCEEDED &&
- dagStatus.getState() != DAGStatus.State.FAILED &&
- dagStatus.getState() != DAGStatus.State.KILLED &&
- dagStatus.getState() != DAGStatus.State.ERROR) {
- if (dagStatus.getState() == DAGStatus.State.RUNNING) {
- ExampleDriver.printDAGStatus(dagClient, vNames);
- }
- try {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- // continue;
- }
- dagStatus = dagClient.getDAGStatus(statusGetOpts);
- } catch (TezException e) {
- LOG.fatal("Failed to get application progress. Exiting");
- return -1;
- }
- }
- ExampleDriver.printDAGStatus(dagClient, vNames,
- true, true);
- LOG.info("DAG " + dagIndex + " completed. "
- + "FinalState=" + dagStatus.getState());
- if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
- LOG.info("DAG " + dagIndex + " diagnostics: "
- + dagStatus.getDiagnostics());
- }
- }
- } catch (Exception e) {
- LOG.error("Error occurred when submitting/running DAGs", e);
- throw e;
- } finally {
- if (!retainStagingDir) {
- pathFs.delete(stagingDir, true);
- }
- LOG.info("Shutting down session");
- tezSession.stop();
- }
-
- if (!useTezSession) {
- ExampleDriver.printDAGStatus(dagClient, vNames);
- LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
- }
- return dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1;
- }
-
- private static void waitForTezSessionReady(TezClient tezSession)
- throws IOException, TezException, InterruptedException {
- tezSession.waitTillReady();
- }
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new TestOrderedWordCount(), args);
- System.exit(res);
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java
deleted file mode 100644
index a9b23e0..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java
+++ /dev/null
@@ -1,295 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tez.mapreduce.examples;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.Map;
-import java.util.StringTokenizer;
-import java.util.TreeMap;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.FileAlreadyExistsException;
-import org.apache.hadoop.mapred.TextInputFormat;
-import org.apache.hadoop.mapred.TextOutputFormat;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.tez.client.TezClient;
-import org.apache.tez.dag.api.DataSinkDescriptor;
-import org.apache.tez.dag.api.DataSourceDescriptor;
-import org.apache.tez.dag.api.GroupInputEdge;
-import org.apache.tez.dag.api.VertexGroup;
-import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.Edge;
-import org.apache.tez.dag.api.InputDescriptor;
-import org.apache.tez.dag.api.ProcessorDescriptor;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.api.Vertex;
-import org.apache.tez.dag.api.client.DAGClient;
-import org.apache.tez.dag.api.client.DAGStatus;
-import org.apache.tez.dag.api.client.StatusGetOpts;
-import org.apache.tez.mapreduce.input.MRInput;
-import org.apache.tez.mapreduce.input.MRInput.MRInputConfigurer;
-import org.apache.tez.mapreduce.output.MROutput;
-import org.apache.tez.mapreduce.processor.SimpleMRProcessor;
-import org.apache.tez.runtime.api.LogicalInput;
-import org.apache.tez.runtime.api.Output;
-import org.apache.tez.runtime.api.ProcessorContext;
-import org.apache.tez.runtime.library.api.KeyValueReader;
-import org.apache.tez.runtime.library.api.KeyValueWriter;
-import org.apache.tez.runtime.library.api.KeyValuesReader;
-import org.apache.tez.runtime.library.conf.OrderedPartitionedKVEdgeConfigurer;
-import org.apache.tez.runtime.library.input.ConcatenatedMergedKeyValuesInput;
-import org.apache.tez.runtime.library.partitioner.HashPartitioner;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
-
-public class UnionExample {
-
- public static class TokenProcessor extends SimpleMRProcessor {
- IntWritable one = new IntWritable(1);
- Text word = new Text();
-
- public TokenProcessor(ProcessorContext context) {
- super(context);
- }
-
- @Override
- public void run() throws Exception {
- Preconditions.checkArgument(getInputs().size() == 1);
- boolean inUnion = true;
- if (getContext().getTaskVertexName().equals("map3")) {
- inUnion = false;
- }
- Preconditions.checkArgument(getOutputs().size() == (inUnion ? 2 : 1));
- Preconditions.checkArgument(getOutputs().containsKey("checker"));
- MRInput input = (MRInput) getInputs().values().iterator().next();
- KeyValueReader kvReader = input.getReader();
- Output output = getOutputs().get("checker");
- KeyValueWriter kvWriter = (KeyValueWriter) output.getWriter();
- MROutput parts = null;
- KeyValueWriter partsWriter = null;
- if (inUnion) {
- parts = (MROutput) getOutputs().get("parts");
- partsWriter = parts.getWriter();
- }
- while (kvReader.next()) {
- StringTokenizer itr = new StringTokenizer(kvReader.getCurrentValue().toString());
- while (itr.hasMoreTokens()) {
- word.set(itr.nextToken());
- kvWriter.write(word, one);
- if (inUnion) {
- partsWriter.write(word, one);
- }
- }
- }
- }
-
- }
-
- public static class UnionProcessor extends SimpleMRProcessor {
- IntWritable one = new IntWritable(1);
-
- public UnionProcessor(ProcessorContext context) {
- super(context);
- }
-
- @Override
- public void run() throws Exception {
- Preconditions.checkArgument(getInputs().size() == 2);
- Preconditions.checkArgument(getOutputs().size() == 2);
- MROutput out = (MROutput) getOutputs().get("union");
- MROutput allParts = (MROutput) getOutputs().get("all-parts");
- KeyValueWriter kvWriter = out.getWriter();
- KeyValueWriter partsWriter = allParts.getWriter();
- Map<String, AtomicInteger> unionKv = Maps.newHashMap();
- LogicalInput union = getInputs().get("union");
- KeyValuesReader kvReader = (KeyValuesReader) union.getReader();
- while (kvReader.next()) {
- String word = ((Text) kvReader.getCurrentKey()).toString();
- IntWritable intVal = (IntWritable) kvReader.getCurrentValues().iterator().next();
- for (int i = 0; i < intVal.get(); ++i) {
- partsWriter.write(word, one);
- }
- AtomicInteger value = unionKv.get(word);
- if (value == null) {
- unionKv.put(word, new AtomicInteger(intVal.get()));
- } else {
- value.addAndGet(intVal.get());
- }
- }
- LogicalInput map3 = getInputs().get("map3");
- kvReader = (KeyValuesReader) map3.getReader();
- while (kvReader.next()) {
- String word = ((Text) kvReader.getCurrentKey()).toString();
- IntWritable intVal = (IntWritable) kvReader.getCurrentValues().iterator().next();
- AtomicInteger value = unionKv.get(word);
- if (value == null) {
- throw new TezUncheckedException("Expected to exist: " + word);
- } else {
- value.getAndAdd(intVal.get() * -2);
- }
- }
- for (AtomicInteger value : unionKv.values()) {
- if (value.get() != 0) {
- throw new TezUncheckedException("Unexpected non-zero value");
- }
- }
- kvWriter.write("Union", new IntWritable(unionKv.size()));
- }
-
- }
-
- private DAG createDAG(FileSystem fs, TezConfiguration tezConf,
- Map<String, LocalResource> localResources, Path stagingDir,
- String inputPath, String outputPath) throws IOException {
- DAG dag = new DAG("UnionExample");
-
- int numMaps = -1;
- Configuration inputConf = new Configuration(tezConf);
- MRInputConfigurer configurer = MRInput.createConfigurer(inputConf, TextInputFormat.class,
- inputPath);
- DataSourceDescriptor dataSource = configurer.generateSplitsInAM(false).create();
-
- Vertex mapVertex1 = new Vertex("map1", new ProcessorDescriptor(
- TokenProcessor.class.getName()), numMaps).addDataSource("MRInput", dataSource);
-
- Vertex mapVertex2 = new Vertex("map2", new ProcessorDescriptor(
- TokenProcessor.class.getName()), numMaps).addDataSource("MRInput", dataSource);
-
- Vertex mapVertex3 = new Vertex("map3", new ProcessorDescriptor(
- TokenProcessor.class.getName()), numMaps).addDataSource("MRInput", dataSource);
-
- Vertex checkerVertex = new Vertex("checker", new ProcessorDescriptor(
- UnionProcessor.class.getName()), 1);
-
- Configuration outputConf = new Configuration(tezConf);
- DataSinkDescriptor od = MROutput.createConfigurer(outputConf,
- TextOutputFormat.class, outputPath).create();
- checkerVertex.addDataSink("union", od);
-
-
- Configuration allPartsConf = new Configuration(tezConf);
- DataSinkDescriptor od2 = MROutput.createConfigurer(allPartsConf,
- TextOutputFormat.class, outputPath + "-all-parts").create();
- checkerVertex.addDataSink("all-parts", od2);
-
- Configuration partsConf = new Configuration(tezConf);
- DataSinkDescriptor od1 = MROutput.createConfigurer(partsConf,
- TextOutputFormat.class, outputPath + "-parts").create();
- VertexGroup unionVertex = dag.createVertexGroup("union", mapVertex1, mapVertex2);
- unionVertex.addDataSink("parts", od1);
-
- OrderedPartitionedKVEdgeConfigurer edgeConf = OrderedPartitionedKVEdgeConfigurer
- .newBuilder(Text.class.getName(), IntWritable.class.getName(),
- HashPartitioner.class.getName()).build();
-
- dag.addVertex(mapVertex1)
- .addVertex(mapVertex2)
- .addVertex(mapVertex3)
- .addVertex(checkerVertex)
- .addEdge(
- new Edge(mapVertex3, checkerVertex, edgeConf.createDefaultEdgeProperty()))
- .addEdge(
- new GroupInputEdge(unionVertex, checkerVertex, edgeConf.createDefaultEdgeProperty(),
- new InputDescriptor(
- ConcatenatedMergedKeyValuesInput.class.getName())));
- return dag;
- }
-
- private static void printUsage() {
- System.err.println("Usage: " + " unionexample <in1> <out1>");
- }
-
- public boolean run(String inputPath, String outputPath, Configuration conf) throws Exception {
- System.out.println("Running UnionExample");
- // conf and UGI
- TezConfiguration tezConf;
- if (conf != null) {
- tezConf = new TezConfiguration(conf);
- } else {
- tezConf = new TezConfiguration();
- }
- UserGroupInformation.setConfiguration(tezConf);
- String user = UserGroupInformation.getCurrentUser().getShortUserName();
-
- // staging dir
- FileSystem fs = FileSystem.get(tezConf);
- String stagingDirStr = Path.SEPARATOR + "user" + Path.SEPARATOR
- + user + Path.SEPARATOR+ ".staging" + Path.SEPARATOR
- + Path.SEPARATOR + Long.toString(System.currentTimeMillis());
- Path stagingDir = new Path(stagingDirStr);
- tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr);
- stagingDir = fs.makeQualified(stagingDir);
-
-
- // No need to add jar containing this class as assumed to be part of
- // the tez jars.
-
- // TEZ-674 Obtain tokens based on the Input / Output paths. For now assuming staging dir
- // is the same filesystem as the one used for Input/Output.
-
- TezClient tezSession = new TezClient("UnionExampleSession", tezConf);
- tezSession.start();
-
- DAGClient dagClient = null;
-
- try {
- if (fs.exists(new Path(outputPath))) {
- throw new FileAlreadyExistsException("Output directory "
- + outputPath + " already exists");
- }
-
- Map<String, LocalResource> localResources =
- new TreeMap<String, LocalResource>();
-
- DAG dag = createDAG(fs, tezConf, localResources,
- stagingDir, inputPath, outputPath);
-
- tezSession.waitTillReady();
- dagClient = tezSession.submitDAG(dag);
-
- // monitoring
- DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(EnumSet.of(StatusGetOpts.GET_COUNTERS));
- if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
- System.out.println("DAG diagnostics: " + dagStatus.getDiagnostics());
- return false;
- }
- return true;
- } finally {
- fs.delete(stagingDir, true);
- tezSession.stop();
- }
- }
-
- public static void main(String[] args) throws Exception {
- if (args.length != 2) {
- printUsage();
- System.exit(2);
- }
- UnionExample job = new UnionExample();
- job.run(args[0], args[1], null);
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/helpers/SplitsInClientOptionParser.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/helpers/SplitsInClientOptionParser.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/helpers/SplitsInClientOptionParser.java
deleted file mode 100644
index 93ec860..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/helpers/SplitsInClientOptionParser.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples.helpers;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-
-import com.google.common.base.Preconditions;
-
-public class SplitsInClientOptionParser {
-
- private CommandLine cmdLine;
- private String[] otherArgs;
-
- private boolean parsed = false;
-
- public SplitsInClientOptionParser() {
-
- }
-
- public String[] getRemainingArgs() {
- Preconditions.checkState(parsed,
- "Cannot get remaining args without parsing");
- return otherArgs;
- }
-
- @SuppressWarnings("static-access")
- public boolean parse(String[] args, boolean defaultVal) throws ParseException {
- Preconditions.checkState(parsed == false,
- "Craete a new instance for different option sets");
- parsed = true;
- Options opts = new Options();
- Option opt = OptionBuilder
- .withArgName("splits_in_client")
- .hasArg()
- .withDescription(
- "specify whether splits should be generated in the client")
- .create("generateSplitsInClient");
- opts.addOption(opt);
- CommandLineParser parser = new GnuParser();
-
- cmdLine = parser.parse(opts, args, false);
- if (cmdLine.hasOption("generateSplitsInClient")) {
- defaultVal = Boolean.parseBoolean(cmdLine
- .getOptionValue("generateSplitsInClient"));
- }
- otherArgs = cmdLine.getArgs();
- return defaultVal;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/package.html
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/package.html b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/package.html
deleted file mode 100644
index 0906086..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/package.html
+++ /dev/null
@@ -1,23 +0,0 @@
-<html>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<body>
-Hadoop example code.
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/2009-write-up/.gitignore
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/2009-write-up/.gitignore b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/2009-write-up/.gitignore
deleted file mode 100644
index 8239358..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/2009-write-up/.gitignore
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-Yahoo2009.aux
-Yahoo2009.bbl
-Yahoo2009.blg
-Yahoo2009.log
-Yahoo2009.out
-Yahoo2009.pdf
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/2009-write-up/Yahoo2009.tex
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/2009-write-up/Yahoo2009.tex b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/2009-write-up/Yahoo2009.tex
deleted file mode 100644
index 1deb812..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/2009-write-up/Yahoo2009.tex
+++ /dev/null
@@ -1,370 +0,0 @@
-% Licensed to the Apache Software Foundation (ASF) under one
-% or more contributor license agreements. See the NOTICE file
-% distributed with this work for additional information
-% regarding copyright ownership. The ASF licenses this file
-% to you under the Apache License, Version 2.0 (the
-% "License"); you may not use this file except in compliance
-% with the License. You may obtain a copy of the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS,
-% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-% See the License for the specific language governing permissions and
-% limitations under the License.
-\documentclass{article}
-\usepackage[pdftex]{hyperref}
-\usepackage[pdftex]{graphicx}
-
-\title{Winning a 60 Second Dash with a Yellow Elephant}
-\author{\href{http://people.apache.org/~omalley}{Owen O'Malley} and
- \href{http://people.apache.org/~acmurthy}{Arun C. Murthy}\\
-\href{http://www.yahoo.com/}{Yahoo!}\\
-owen@yahoo-inc.com and acm@yahoo-inc.com}
-\date{April 2009}
-\begin{document}
-\maketitle
-\href{http://hadoop.apache.org/core}{Apache Hadoop} is a open source
-software framework that dramatically simplifies writing distributed
-data intensive applications. It provides a distributed file system,
-which is modeled after the Google File System\cite{gfs}, and a
-map/reduce\cite{mapreduce} implementation that manages distributed
-computation. Jim Gray defined a benchmark to compare large sorting
-programs. Since the core of map/reduce is a distributed sort, most of
-the custom code is glue to get the desired behavior.
-
-\section{Benchmark Rules}
-
-Jim's Gray's sort benchmark consists of a set of many related
-benchmarks, each with their own rules. All of the sort benchmarks
-measure the time to sort different numbers of 100 byte records. The
-first 10 bytes of each record is the key and the rest is the
-value. The \textbf{minute sort} must finish end to end in less than a
-minute. The \textbf{Gray sort} must sort more than 100 terabytes and
-must run for at least an hour.
-
-\begin{itemize}
-\item The input data must precisely match the data generated by the C
- data generator.
-\item The input must not be in the operating system's file
- cache when the job starts.. Under Linux, this requires using the memory for something
- else between sorting runs.
-\item The input and output data must not be compressed.
-\item The output must not overwrite the input.
-\item The output must be synced to disk.
-\item The 128 bit sum of the crc32's of each key/value pair must be
- calculated for the input and output. Naturally, they must be
- identical.
-\item The output may be divided into multiple output files, but it
- must be totally ordered (simply concatenating the output files must
- produce the completely sorted output).
-\item Starting and distributing the application to the cluster must be
- included in the execution time.
-\item Any sampling must be included in the execution time.
-\end{itemize}
-
-\section{Hadoop implementation}
-
-We extended the programs from last year to create and manipulate the
-new binary format and match the new rules. There are now 4 Hadoop
-map/reduce applications to support the benchmark:
-\begin{enumerate}
-\item \textbf{TeraGen} is a map/reduce program to generate the data.
-\item \textbf{TeraSort} samples the input data and uses map/reduce to
- sort the data into a total order.
-\item \textbf{TeraSum} is a map/reduce program computes the 128 bit
- sum of the crc32 of each key/value pair.
-\item \textbf{TeraValidate} is a map/reduce program that validates the
- output is sorted and computes the sum of the checksums as TeraSum.
-\end{enumerate}
-The update to the terasort programs will be checked in as
-\href{http://issues.apache.org/jira/browse/HADOOP-5716}{HADOOP-5716}.
-
-\textbf{TeraGen} generates input data for the sort that is byte for byte
-equivalent to the C version that was released in March of 2009,
-including specific keys and values. It divides the desired number of
-rows by the desired number of tasks and assigns ranges of rows to each
-map. The map jumps the random number generator to the correct value
-for the first row and generates the following rows.
-
-\textbf{TeraSort} is a standard map/reduce sort, except for a custom
-partitioner that ensures that all of the keys in reduce $N$ are after
-all of the keys in reduce $N-1$. This is a requirement of the contest
-so that the output of the sort is totally ordered, even if it is
-divided up by reduce.
-
-We wrote an input and output format, used by all 4 applications to
-read and write the files in the new format.
-
-\textbf{TeraSum} computes the 128 bit sum of the CRC32 of each
-key/value pair. Each map computes the sum of its input and emits a
-single 128 bit sum. There is a single reduce that adds the sums from
-each map. We used this program on the input directory to calculate the
-sum of the checksums of each key/value pair to check the correctness
-of the output of the sort. We also used TeraSum on a distinct dataset
-that was larger than the total RAM in the cluster to flush the Linux
-file cache between runs of the small (500 GB and 1TB) sorts.
-
-\textbf{TeraValidate} ensures that the output is globally sorted. It
-creates one map per file in the output directory and each map
-ensures that each key is less than or equal to the previous one. The
-map also generates records with the first and last keys of the file
-and the reduce ensures that the first key of file $i$ is greater that
-the last key of file $i-1$. Any problems are reported as output of the
-reduce with the keys that are out of order. Additionally, TeraValidate
-calculates the sum of checksums of the output directory.
-
-\section{Hardware and Operating System}
-
-We ran our benchmarks on Yahoo's Hammer cluster. Hammer's hardware is
-very similar to the hardware that we used in last year's terabyte
-sort. The hardware and operating system details are:
-
-\begin{itemize}
-\item approximately 3800 nodes (in such a large cluster, nodes are
- always down)
-\item 2 quad core Xeons @ 2.5ghz per node
-\item 4 SATA disks per node
-\item 8G RAM per node (upgraded to 16GB before the petabyte sort)
-\item 1 gigabit ethernet on each node
-\item 40 nodes per rack
-\item 8 gigabit ethernet uplinks from each rack to the core
-\item Red Hat Enterprise Linux Server Release 5.1 (kernel 2.6.18)
-\item Sun Java JDK (1.6.0\_05-b13 and 1.6.0\_13-b03) (32 and 64 bit)
-\end{itemize}
-
-We hit a JVM bug in 1.6.0\_05-b13 on the larger sorts (100TB and 1PB)
-and switched over to the later JVM, which resolved the issue. For the
-larger sorts, we used 64 bit JVMs for the Name Node and Job Tracker.
-
-\section{Software and Configuration}
-
-The version of Hadoop we used was a private branch of trunk that was
-started in January 2009, which is after the 0.20 branch was feature
-frozen. We used git to manage our branch and it allowed us to easily
-coordinate our work, track our changes, and resynchronize with the
-current Hadoop trunk.
-
-The changes include:
-
-\begin{enumerate}
-
-\item Updated the terasort example in the Hadoop code base to match
- the dataset defined by the rule changes in the benchmark from March
- of 2009.
- (\href{http://issues.apache.org/jira/browse/HADOOP-5716}{HADOOP-5716})
-
-\item We reimplemented the reducer side of Hadoop's shuffle. The
- redesign improved the performance of the shuffle and removed
- bottlenecks and over-throttling. It also made the code more
- maintainable and understandable by breaking a 3000 line Java file
- into multiple classes with a clean set of interfaces.
- (\href{http://issues.apache.org/jira/browse/HADOOP-5223}{HADOOP-5223})
-
-\item The new shuffle also fetches multiple map outputs from the same
- node over each connection rather than one at a time. Fetching
- multiple map outputs at the same time avoids connection setup costs
- and also avoids the round trip while the server responds to the request.
- (\href{http://issues.apache.org/jira/browse/HADOOP-1338}{HADOOP-1338})
-
-\item Allowed configuring timeouts on the shuffle connections and we
- shortened them for the small sorts. We observed cases where the
- connections for the shuffle would hang until the timeout, which made
- low latency jobs impossibly long.
- (\href{http://issues.apache.org/jira/browse/HADOOP-5789}{HADOOP-5789})
-
-\item Set TCP no-delay and more frequent pings between the Task and
- the Task Tracker to reduce latency in detecting problems.
- (\href{http://issues.apache.org/jira/browse/HADOOP-5788}{HADOOP-5788})
-
-\item We added some protection code to detect incorrect data being
- transmitted in the shuffle from causing the reduce to fail. It
- appears this is either a JVM NIO bug or Jetty bug that likely
- affects 0.20 and trunk under heavy load.
- (\href{http://issues.apache.org/jira/browse/HADOOP-5783}{HADOOP-5783})
-
-\item We used LZO compression on the map outputs. On the new dataset, LZO
- compresses down to 45\% of the original size. By comparison, the
- dataset from last year compresses to 20\% of the original size. Last
- year, the shuffle would run out of direct buffers if we used
- compression on the map outputs.
-
-\item We implemented memory to memory merges in the reduce during the
- shuffle to combine the map outputs in memory before we finish the
- shuffle, thereby reducing the work needed when the reduce is
- running.
-
-\item We multi-threaded the sampling code that read the input set to
- find the partition points between the reduces. We also wrote a
- simple partitioner that assumes the keys are evenly
- distributed. Since the new dataset does not require sampling, the
- simple partitioner produces very even partitions.
- (\href{http://issues.apache.org/jira/browse/HADOOP-4946}{HADOOP-4946})
-
-\item On the smaller clusters, we configured the system with faster
- heartbeat cycles from the Task Trackers to the Job Tracker (it
- defaults to 10 secs / 1000 nodes, but we made it configurable and
- brought it down to 2 seconds/1000 nodes to provide lower latency)
- (\href{http://issues.apache.org/jira/browse/HADOOP-5784}{HADOOP-5784})
-
-\item Typically the Job Tracker assigns tasks to Task Trackers on a
- first come first served basis. This greedy assignment of tasks did
- not lead to good data locality. However, by taking a global view and
- placing all of the map tasks at once, the system achieves much better
- locality. Rather than implement global scheduling for all of Hadoop,
- which would be much harder, we implemented a global scheduler for
- the terasort example in the input format. Basically, the input
- format computes the splits and assigns work to the nodes that have
- the fewest blocks first. For a node that has more blocks
- than map slots, it picks the block that have the fewest remaining
- locations left. This greedy global algorithm seems to get very good
- locality. The input format would schedule the maps and then change
- the input split descriptions to only have a single location instead
- of the original 3. This increased task locality by 40\% or so over
- the greedy scheduler.
-
-\item Hadoop 0.20 added setup and cleanup tasks. Since they are not
- required for the sort benchmarks, we allow them to be disabled to
- reduce the latency of starting and stopping the job.
- (\href{http://issues.apache.org/jira/browse/HADOOP-5785}{HADOOP-5785})
-
-\item We discovered a performance problem where in some contexts the
- cost of using the JNI-based CRC32 was very high. By implementing it
- in pure Java, the average case is a little slower, but the worst
- case is much better.
- (\href{http://issues.apache.org/jira/browse/HADOOP-5598}{HADOOP-5598})
-
-\item We found and removed some hard-coded wait loops from the
- framework that don't matter for large jobs, but can seriously slow
- down low latency jobs.
-
-\item Allowed setting the logging level for the tasks, so that we
- could cut down on logging. When running for "real" we configure the
- logging level to WARN instead of the default INFO. Reducing the
- amount of logging has a huge impact on the performance of the
- system, but obviously makes debugging and analysis much harder.
- (\href{http://issues.apache.org/jira/browse/HADOOP-5786}{HADOOP-5786})
-
-\item One optimization that we didn't finish is to optimize the job
- planning code. Currently, it uses an RPC to the Name Node for each
- input file, which we have observed taking a substantial amount of
- time. For the terabyte sort, our investigations show that we
- could save about 4 seconds out of the 8 that were spent on setting
- up the job.
- (\href{http://issues.apache.org/jira/browse/HADOOP-5795}{HADOOP-5795})
-
-\end{enumerate}
-
-\section{Results}
-
-Hadoop has made a lot of progress in the last year and we were able to
-run much lower latency jobs as well as much larger jobs. Note that in
-any large cluster and distributed application, there are a lot of
-moving pieces and thus we have seen a wide variation in execution
-times. As Hadoop evolves and becomes more graceful in the presence of
-hardware degradation and failure, this variation should smooth
-out. The best times for each of the listed sort sizes were:
-\\
-
-\begin{tabular}{| c | c | c | c | c | c |}
-\hline
-Bytes & Nodes & Maps & Reduces & Replication & Time \\
-\hline
-$5*10^{11}$ & 1406 & 8000 & 2600 & 1 & 59 seconds \\
-$10^{12}$ & 1460 & 8000 & 2700 & 1 & 62 seconds \\
-$10^{14}$ & 3452 & 190,000 & 10,000 & 2 & 173 minutes \\
-$10^{15}$ & 3658 & 80,000 & 20,000 & 2 & 975 minutes \\
-\hline
-\end{tabular}\\
-
-Within the rules for the 2009 Gray sort, our 500 GB sort set a new
-record for the minute sort and the 1PB sort set a new record of 1.03
-TB/minute. The 62 second terabyte sort would have set a new record,
-but the terabyte benchmark that we won last year has been
-retired. (Clearly the minute sort and terabyte sort are rapidly
-converging, and thus it is not a loss.) One piece of trivia is that
-only the petabyte dataset had any duplicate keys (40 of them).
-
-Because the smaller sorts needed lower latency and faster network, we
-only used part of the cluster for those runs. In particular, instead
-of our normal 5:1 over subscription between racks, we limited it to 16
-nodes in each rack for a 2:1 over subscription. The smaller runs can
-also use output replication of 1, because they only take minutes to
-run and run on smaller clusters, the likelihood of a node failing is
-fairly low. On the larger runs, failure is expected and thus
-replication of 2 is required. HDFS protects against data loss during
-rack failure by writing the second replica on a different rack and
-thus writing the second replica is relatively slow.
-
-We've included the timelines for the jobs counting from the job
-submission at the Job Tracker. The diagrams show the number of tasks
-running at each point in time. While maps only have a single phase,
-the reduces have three: \textbf{shuffle}, \textbf{merge}, and
-\textbf{reduce}. The shuffle is the transfer of the data from the
-maps. Merge doesn't happen in these benchmarks, because none of the
-reduces need multiple levels of merges. Finally, the reduce phase is
-where the final merge and writing to HDFS happens. I've also included
-a category named \textbf{waste} that represents task attempts that
-were running, but ended up either failing, or being killed (often as
-speculatively executed task attempts). The job logs and configuration
-for the four runs, which are the raw data for the charts, are
-available on
-\href{http://people.apache.org/~omalley/tera-2009/}{http://people.apache.org/~omalley/tera-2009/}.
-
-If you compare this years charts to last year's, you'll notice that
-tasks are launching much faster now. Last year we only launched one
-task per heartbeat, so it took 40 seconds to get all of the tasks
-launched. Now, Hadoop will fill up a Task Tracker in a single
-heartbeat. Reducing that job launch overhead is very important
-for getting runs under a minute.
-
-As with last year, we ran with significantly larger tasks than the
-defaults for Hadoop. Even with the new more aggressive shuffle,
-minimizing the number of transfers (maps * reduces) is very important
-to the performance of the job. Notice that in the petabyte sort, each
-map is processing 15 GB instead of the default 128 MB and each reduce
-is handling 50 GB. When we ran the petabyte with more typical values
-1.5 GB / map, it took 40 hours to finish. Therefore, to increase
-throughput, it makes sense to consider increasing the default block
-size, which translates into the default map size, to at least up to 1
-GB.
-
-\section{Comments on the Rule Changes}
-
-The group that runs the Gray Sort Benchmark made very substantial
-changes to the rules this year. The changes were not announced; but
-rather appeared on the website in March. We feel that it was too late
-to make rule changes and that the benchmark should have been changed
-next year. We'd also like to point out that while most of the changes to
-the data generator were positive, it was a poor choice to remove the
-skewed distribution of the keys. The previously skewed distribution
-required sampling of the input to pick good partition points between
-the reduces. The current dataset picks keys so completely random that
-sampling is counter productive and yields even less distributions between the
-reduces.
-
-\bibliographystyle{abbrv}
-\bibliography{tera}
-
-\begin{figure}[!p]
-\includegraphics[width=4.21in]{500GBTaskTime.png}
-\caption{500 GB sort tasks across time}\label{500GbTimeline}
-\end{figure}
-
-\begin{figure}[!p]
-\includegraphics[width=4.5in]{1TBTaskTime.png}
-\caption{1 TB sort tasks across time}\label{1TbTimeline}
-\end{figure}
-
-\begin{figure}[!p]
-\includegraphics[width=4.5in]{100TBTaskTime.png}
-\caption{100 TB sort tasks across time}\label{100TbTimeline}
-\end{figure}
-
-\begin{figure}[!p]
-\includegraphics[width=4.5in]{1PBTaskTime.png}
-\caption{1 PB sort tasks across time}\label{1PbTimeline}
-\end{figure}
-
-\end{document}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/2009-write-up/tera.bib
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/2009-write-up/tera.bib b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/2009-write-up/tera.bib
deleted file mode 100644
index 50a797f..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/2009-write-up/tera.bib
+++ /dev/null
@@ -1,31 +0,0 @@
-% Licensed to the Apache Software Foundation (ASF) under one
-% or more contributor license agreements. See the NOTICE file
-% distributed with this work for additional information
-% regarding copyright ownership. The ASF licenses this file
-% to you under the Apache License, Version 2.0 (the
-% "License"); you may not use this file except in compliance
-% with the License. You may obtain a copy of the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS,
-% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-% See the License for the specific language governing permissions and
-% limitations under the License.
-@INPROCEEDINGS{mapreduce,
- AUTHOR = "Jeffery Dean and Sanjay Ghemawat",
- TITLE = "MapReduce: Simplified Data Processing on Large Clusters",
- BOOKTITLE = "Sixth Symposium on Operating System Design and Implementation",
- MONTH = "December",
- ADDRESS = "San Francisco, CA",
- YEAR = {2004} }
-
-@INPROCEEDINGS{gfs,
- AUTHOR = "Sanjay Ghemawat and Howard Gobioff and Shun-Tak Leung",
- TITLE = "The Google File System",
- BOOKTITLE = "19th Symposium on Operating Systems Principles",
- ORGANIZATION = "ACM",
- MONTH = "October",
- ADDRESS = "Lake George, NY",
- YEAR = {2003} }
[06/10] TEZ-1055. Rename tez-mapreduce-examples to tez-examples
(Hitesh Shah via bikas)
Posted by bi...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/GenSort.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/GenSort.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/GenSort.java
deleted file mode 100644
index c8517e5..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/GenSort.java
+++ /dev/null
@@ -1,251 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tez.mapreduce.examples.terasort;
-
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintStream;
-import java.math.BigInteger;
-import java.util.zip.Checksum;
-
-import org.apache.hadoop.util.PureJavaCrc32;
-
-/**
- * A single process data generator for the terasort data. Based on gensort.c
- * version 1.1 (3 Mar 2009) from Chris Nyberg <ch...@ordinal.com>.
- */
-public class GenSort {
-
- /**
- * Generate a "binary" record suitable for all sort benchmarks *except*
- * PennySort.
- */
- static void generateRecord(byte[] recBuf, Unsigned16 rand,
- Unsigned16 recordNumber) {
- /* generate the 10-byte key using the high 10 bytes of the 128-bit
- * random number
- */
- for(int i=0; i < 10; ++i) {
- recBuf[i] = rand.getByte(i);
- }
-
- /* add 2 bytes of "break" */
- recBuf[10] = 0x00;
- recBuf[11] = 0x11;
-
- /* convert the 128-bit record number to 32 bits of ascii hexadecimal
- * as the next 32 bytes of the record.
- */
- for (int i = 0; i < 32; i++) {
- recBuf[12 + i] = (byte) recordNumber.getHexDigit(i);
- }
-
- /* add 4 bytes of "break" data */
- recBuf[44] = (byte) 0x88;
- recBuf[45] = (byte) 0x99;
- recBuf[46] = (byte) 0xAA;
- recBuf[47] = (byte) 0xBB;
-
- /* add 48 bytes of filler based on low 48 bits of random number */
- for(int i=0; i < 12; ++i) {
- recBuf[48+i*4] = recBuf[49+i*4] = recBuf[50+i*4] = recBuf[51+i*4] =
- (byte) rand.getHexDigit(20 + i);
- }
-
- /* add 4 bytes of "break" data */
- recBuf[96] = (byte) 0xCC;
- recBuf[97] = (byte) 0xDD;
- recBuf[98] = (byte) 0xEE;
- recBuf[99] = (byte) 0xFF;
- }
-
-
- private static BigInteger makeBigInteger(long x) {
- byte[] data = new byte[8];
- for(int i=0; i < 8; ++i) {
- data[i] = (byte) (x >>> (56 - 8*i));
- }
- return new BigInteger(1, data);
- }
-
- private static final BigInteger NINETY_FIVE = new BigInteger("95");
-
- /**
- * Generate an ascii record suitable for all sort benchmarks including
- * PennySort.
- */
- static void generateAsciiRecord(byte[] recBuf, Unsigned16 rand,
- Unsigned16 recordNumber) {
-
- /* generate the 10-byte ascii key using mostly the high 64 bits.
- */
- long temp = rand.getHigh8();
- if (temp < 0) {
- // use biginteger to avoid the negative sign problem
- BigInteger bigTemp = makeBigInteger(temp);
- recBuf[0] = (byte) (' ' + (bigTemp.mod(NINETY_FIVE).longValue()));
- temp = bigTemp.divide(NINETY_FIVE).longValue();
- } else {
- recBuf[0] = (byte) (' ' + (temp % 95));
- temp /= 95;
- }
- for(int i=1; i < 8; ++i) {
- recBuf[i] = (byte) (' ' + (temp % 95));
- temp /= 95;
- }
- temp = rand.getLow8();
- if (temp < 0) {
- BigInteger bigTemp = makeBigInteger(temp);
- recBuf[8] = (byte) (' ' + (bigTemp.mod(NINETY_FIVE).longValue()));
- temp = bigTemp.divide(NINETY_FIVE).longValue();
- } else {
- recBuf[8] = (byte) (' ' + (temp % 95));
- temp /= 95;
- }
- recBuf[9] = (byte)(' ' + (temp % 95));
-
- /* add 2 bytes of "break" */
- recBuf[10] = ' ';
- recBuf[11] = ' ';
-
- /* convert the 128-bit record number to 32 bits of ascii hexadecimal
- * as the next 32 bytes of the record.
- */
- for (int i = 0; i < 32; i++) {
- recBuf[12 + i] = (byte) recordNumber.getHexDigit(i);
- }
-
- /* add 2 bytes of "break" data */
- recBuf[44] = ' ';
- recBuf[45] = ' ';
-
- /* add 52 bytes of filler based on low 48 bits of random number */
- for(int i=0; i < 13; ++i) {
- recBuf[46+i*4] = recBuf[47+i*4] = recBuf[48+i*4] = recBuf[49+i*4] =
- (byte) rand.getHexDigit(19 + i);
- }
-
- /* add 2 bytes of "break" data */
- recBuf[98] = '\r'; /* nice for Windows */
- recBuf[99] = '\n';
-}
-
-
- private static void usage() {
- PrintStream out = System.out;
- out.println("usage: gensort [-a] [-c] [-bSTARTING_REC_NUM] NUM_RECS FILE_NAME");
- out.println("-a Generate ascii records required for PennySort or JouleSort.");
- out.println(" These records are also an alternative input for the other");
- out.println(" sort benchmarks. Without this flag, binary records will be");
- out.println(" generated that contain the highest density of randomness in");
- out.println(" the 10-byte key.");
- out.println( "-c Calculate the sum of the crc32 checksums of each of the");
- out.println(" generated records and send it to standard error.");
- out.println("-bN Set the beginning record generated to N. By default the");
- out.println(" first record generated is record 0.");
- out.println("NUM_RECS The number of sequential records to generate.");
- out.println("FILE_NAME The name of the file to write the records to.\n");
- out.println("Example 1 - to generate 1000000 ascii records starting at record 0 to");
- out.println("the file named \"pennyinput\":");
- out.println(" gensort -a 1000000 pennyinput\n");
- out.println("Example 2 - to generate 1000 binary records beginning with record 2000");
- out.println("to the file named \"partition2\":");
- out.println(" gensort -b2000 1000 partition2");
- System.exit(1);
- }
-
-
- public static void outputRecords(OutputStream out,
- boolean useAscii,
- Unsigned16 firstRecordNumber,
- Unsigned16 recordsToGenerate,
- Unsigned16 checksum
- ) throws IOException {
- byte[] row = new byte[100];
- Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
- Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
- Checksum crc = new PureJavaCrc32();
- Unsigned16 tmp = new Unsigned16();
- lastRecordNumber.add(recordsToGenerate);
- Unsigned16 ONE = new Unsigned16(1);
- Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
- while (!recordNumber.equals(lastRecordNumber)) {
- Random16.nextRand(rand);
- if (useAscii) {
- generateAsciiRecord(row, rand, recordNumber);
- } else {
- generateRecord(row, rand, recordNumber);
- }
- if (checksum != null) {
- crc.reset();
- crc.update(row, 0, row.length);
- tmp.set(crc.getValue());
- checksum.add(tmp);
- }
- recordNumber.add(ONE);
- out.write(row);
- }
- }
-
- public static void main(String[] args) throws Exception {
- Unsigned16 startingRecord = new Unsigned16();
- Unsigned16 numberOfRecords;
- OutputStream out;
- boolean useAscii = false;
- Unsigned16 checksum = null;
-
- int i;
- for(i=0; i < args.length; ++i) {
- String arg = args[i];
- int argLength = arg.length();
- if (argLength >= 1 && arg.charAt(0) == '-') {
- if (argLength < 2) {
- usage();
- }
- switch (arg.charAt(1)) {
- case 'a':
- useAscii = true;
- break;
- case 'b':
- startingRecord = Unsigned16.fromDecimal(arg.substring(2));
- break;
- case 'c':
- checksum = new Unsigned16();
- break;
- default:
- usage();
- }
- } else {
- break;
- }
- }
- if (args.length - i != 2) {
- usage();
- }
- numberOfRecords = Unsigned16.fromDecimal(args[i]);
- out = new FileOutputStream(args[i+1]);
-
- outputRecords(out, useAscii, startingRecord, numberOfRecords, checksum);
- out.close();
- if (checksum != null) {
- System.out.println(checksum);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/Random16.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/Random16.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/Random16.java
deleted file mode 100644
index 31cbd48..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/Random16.java
+++ /dev/null
@@ -1,374 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tez.mapreduce.examples.terasort;
-
-/**
- * This class implements a 128-bit linear congruential generator.
- * Specifically, if X0 is the most recently issued 128-bit random
- * number (or a seed of 0 if no random number has already been generated,
- * the next number to be generated, X1, is equal to:
- * X1 = (a * X0 + c) mod 2**128
- * where a is 47026247687942121848144207491837523525
- * or 0x2360ed051fc65da44385df649fccf645
- * and c is 98910279301475397889117759788405497857
- * or 0x4a696d47726179524950202020202001
- * The coefficient "a" is suggested by:
- * Pierre L'Ecuyer, "Tables of linear congruential generators of different
- * sizes and good lattice structure", Mathematics of Computation, 68
- * pp. 249 - 260 (1999)
- * http://www.ams.org/mcom/1999-68-225/S0025-5718-99-00996-5/S0025-5718-99-00996-5.pdf
- * The constant "c" meets the simple suggestion by the same reference that
- * it be odd.
- *
- * There is also a facility for quickly advancing the state of the
- * generator by a fixed number of steps - this facilitates parallel
- * generation.
- *
- * This is based on 1.0 of rand16.c from Chris Nyberg
- * <ch...@ordinal.com>.
- */
-class Random16 {
-
- /**
- * The "Gen" array contain powers of 2 of the linear congruential generator.
- * The index 0 struct contain the "a" coefficient and "c" constant for the
- * generator. That is, the generator is:
- * f(x) = (Gen[0].a * x + Gen[0].c) mod 2**128
- *
- * All structs after the first contain an "a" and "c" that
- * comprise the square of the previous function.
- *
- * f**2(x) = (Gen[1].a * x + Gen[1].c) mod 2**128
- * f**4(x) = (Gen[2].a * x + Gen[2].c) mod 2**128
- * f**8(x) = (Gen[3].a * x + Gen[3].c) mod 2**128
- * ...
-
- */
- private static class RandomConstant {
- final Unsigned16 a;
- final Unsigned16 c;
- public RandomConstant(String left, String right) {
- a = new Unsigned16(left);
- c = new Unsigned16(right);
- }
- }
-
- private static final RandomConstant[] genArray = new RandomConstant[]{
- /* [ 0] */ new RandomConstant("2360ed051fc65da44385df649fccf645",
- "4a696d47726179524950202020202001"),
- /* [ 1] */ new RandomConstant("17bce35bdf69743c529ed9eb20e0ae99",
- "95e0e48262b3edfe04479485c755b646"),
- /* [ 2] */ new RandomConstant("f4dd417327db7a9bd194dfbe42d45771",
- "882a02c315362b60765f100068b33a1c"),
- /* [ 3] */ new RandomConstant("6347af777a7898f6d1a2d6f33505ffe1",
- "5efc4abfaca23e8ca8edb1f2dfbf6478"),
- /* [ 4] */ new RandomConstant("b6a4239f3b315f84f6ef6d3d288c03c1",
- "f25bd15439d16af594c1b1bafa6239f0"),
- /* [ 5] */ new RandomConstant("2c82901ad1cb0cd182b631ba6b261781",
- "89ca67c29c9397d59c612596145db7e0"),
- /* [ 6] */ new RandomConstant("dab03f988288676ee49e66c4d2746f01",
- "8b6ae036713bd578a8093c8eae5c7fc0"),
- /* [ 7] */ new RandomConstant("602167331d86cf5684fe009a6d09de01",
- "98a2542fd23d0dbdff3b886cdb1d3f80"),
- /* [ 8] */ new RandomConstant("61ecb5c24d95b058f04c80a23697bc01",
- "954db923fdb7933e947cd1edcecb7f00"),
- /* [ 9] */ new RandomConstant("4a5c31e0654c28aa60474e83bf3f7801",
- "00be4a36657c98cd204e8c8af7dafe00"),
- /* [ 10] */ new RandomConstant("ae4f079d54fbece1478331d3c6bef001",
- "991965329dccb28d581199ab18c5fc00"),
- /* [ 11] */ new RandomConstant("101b8cb830c7cb927ff1ed50ae7de001",
- "e1a8705b63ad5b8cd6c3d268d5cbf800"),
- /* [ 12] */ new RandomConstant("f54a27fc056b00e7563f3505e0fbc001",
- "2b657bbfd6ed9d632079e70c3c97f000"),
- /* [ 13] */ new RandomConstant("df8a6fc1a833d201f98d719dd1f78001",
- "59b60ee4c52fa49e9fe90682bd2fe000"),
- /* [ 14] */ new RandomConstant("5480a5015f101a4ea7e3f183e3ef0001",
- "cc099c88030679464fe86aae8a5fc000"),
- /* [ 15] */ new RandomConstant("a498509e76e5d7925f539c28c7de0001",
- "06b9abff9f9f33dd30362c0154bf8000"),
- /* [ 16] */ new RandomConstant("0798a3d8b10dc72e60121cd58fbc0001",
- "e296707121688d5a0260b293a97f0000"),
- /* [ 17] */ new RandomConstant("1647d1e78ec02e665fafcbbb1f780001",
- "189ffc4701ff23cb8f8acf6b52fe0000"),
- /* [ 18] */ new RandomConstant("a7c982285e72bf8c0c8ddfb63ef00001",
- "5141110ab208fb9d61fb47e6a5fc0000"),
- /* [ 19] */ new RandomConstant("3eb78ee8fb8c56dbc5d4e06c7de00001",
- "3c97caa62540f2948d8d340d4bf80000"),
- /* [ 20] */ new RandomConstant("72d03b6f4681f2f9fe8e44d8fbc00001",
- "1b25cb9cfe5a0c963174f91a97f00000"),
- /* [ 21] */ new RandomConstant("ea85f81e4f502c9bc8ae99b1f7800001",
- "0c644570b4a487103c5436352fe00000"),
- /* [ 22] */ new RandomConstant("629c320db08b00c6bfa57363ef000001",
- "3d0589c28869472bde517c6a5fc00000"),
- /* [ 23] */ new RandomConstant("c5c4b9ce268d074a386be6c7de000001",
- "bc95e5ab36477e65534738d4bf800000"),
- /* [ 24] */ new RandomConstant("f30bbbbed1596187555bcd8fbc000001",
- "ddb02ff72a031c01011f71a97f000000"),
- /* [ 25] */ new RandomConstant("4a1000fb26c9eeda3cc79b1f78000001",
- "2561426086d9acdb6c82e352fe000000"),
- /* [ 26] */ new RandomConstant("89fb5307f6bf8ce2c1cf363ef0000001",
- "64a788e3c118ed1c8215c6a5fc000000"),
- /* [ 27] */ new RandomConstant("830b7b3358a5d67ea49e6c7de0000001",
- "e65ea321908627cfa86b8d4bf8000000"),
- /* [ 28] */ new RandomConstant("fd8a51da91a69fe1cd3cd8fbc0000001",
- "53d27225604d85f9e1d71a97f0000000"),
- /* [ 29] */ new RandomConstant("901a48b642b90b55aa79b1f780000001",
- "ca5ec7a3ed1fe55e07ae352fe0000000"),
- /* [ 30] */ new RandomConstant("118cdefdf32144f394f363ef00000001",
- "4daebb2e085330651f5c6a5fc0000000"),
- /* [ 31] */ new RandomConstant("0a88c0a91cff430829e6c7de00000001",
- "9d6f1a00a8f3f76e7eb8d4bf80000000"),
- /* [ 32] */ new RandomConstant("433bef4314f16a9453cd8fbc00000001",
- "158c62f2b31e496dfd71a97f00000000"),
- /* [ 33] */ new RandomConstant("c294b02995ae6738a79b1f7800000001",
- "290e84a2eb15fd1ffae352fe00000000"),
- /* [ 34] */ new RandomConstant("913575e0da8b16b14f363ef000000001",
- "e3dc1bfbe991a34ff5c6a5fc00000000"),
- /* [ 35] */ new RandomConstant("2f61b9f871cf4e629e6c7de000000001",
- "ddf540d020b9eadfeb8d4bf800000000"),
- /* [ 36] */ new RandomConstant("78d26ccbd68320c53cd8fbc000000001",
- "8ee4950177ce66bfd71a97f000000000"),
- /* [ 37] */ new RandomConstant("8b7ebd037898518a79b1f78000000001",
- "39e0f787c907117fae352fe000000000"),
- /* [ 38] */ new RandomConstant("0b5507b61f78e314f363ef0000000001",
- "659d2522f7b732ff5c6a5fc000000000"),
- /* [ 39] */ new RandomConstant("4f884628f812c629e6c7de0000000001",
- "9e8722938612a5feb8d4bf8000000000"),
- /* [ 40] */ new RandomConstant("be896744d4a98c53cd8fbc0000000001",
- "e941a65d66b64bfd71a97f0000000000"),
- /* [ 41] */ new RandomConstant("daf63a553b6318a79b1f780000000001",
- "7b50d19437b097fae352fe0000000000"),
- /* [ 42] */ new RandomConstant("2d7a23d8bf06314f363ef00000000001",
- "59d7b68e18712ff5c6a5fc0000000000"),
- /* [ 43] */ new RandomConstant("392b046a9f0c629e6c7de00000000001",
- "4087bab2d5225feb8d4bf80000000000"),
- /* [ 44] */ new RandomConstant("eb30fbb9c218c53cd8fbc00000000001",
- "b470abc03b44bfd71a97f00000000000"),
- /* [ 45] */ new RandomConstant("b9cdc30594318a79b1f7800000000001",
- "366630eaba897fae352fe00000000000"),
- /* [ 46] */ new RandomConstant("014ab453686314f363ef000000000001",
- "a2dfc77e8512ff5c6a5fc00000000000"),
- /* [ 47] */ new RandomConstant("395221c7d0c629e6c7de000000000001",
- "1e0d25a14a25feb8d4bf800000000000"),
- /* [ 48] */ new RandomConstant("4d972813a18c53cd8fbc000000000001",
- "9d50a5d3944bfd71a97f000000000000"),
- /* [ 49] */ new RandomConstant("06f9e2374318a79b1f78000000000001",
- "bf7ab5eb2897fae352fe000000000000"),
- /* [ 50] */ new RandomConstant("bd220cae86314f363ef0000000000001",
- "925b14e6512ff5c6a5fc000000000000"),
- /* [ 51] */ new RandomConstant("36fd3a5d0c629e6c7de0000000000001",
- "724cce0ca25feb8d4bf8000000000000"),
- /* [ 52] */ new RandomConstant("60def8ba18c53cd8fbc0000000000001",
- "1af42d1944bfd71a97f0000000000000"),
- /* [ 53] */ new RandomConstant("8d500174318a79b1f780000000000001",
- "0f529e32897fae352fe0000000000000"),
- /* [ 54] */ new RandomConstant("48e842e86314f363ef00000000000001",
- "844e4c6512ff5c6a5fc0000000000000"),
- /* [ 55] */ new RandomConstant("4af185d0c629e6c7de00000000000001",
- "9f40d8ca25feb8d4bf80000000000000"),
- /* [ 56] */ new RandomConstant("7a670ba18c53cd8fbc00000000000001",
- "9912b1944bfd71a97f00000000000000"),
- /* [ 57] */ new RandomConstant("86de174318a79b1f7800000000000001",
- "9c69632897fae352fe00000000000000"),
- /* [ 58] */ new RandomConstant("55fc2e86314f363ef000000000000001",
- "e1e2c6512ff5c6a5fc00000000000000"),
- /* [ 59] */ new RandomConstant("ccf85d0c629e6c7de000000000000001",
- "68058ca25feb8d4bf800000000000000"),
- /* [ 60] */ new RandomConstant("1df0ba18c53cd8fbc000000000000001",
- "610b1944bfd71a97f000000000000000"),
- /* [ 61] */ new RandomConstant("4be174318a79b1f78000000000000001",
- "061632897fae352fe000000000000000"),
- /* [ 62] */ new RandomConstant("d7c2e86314f363ef0000000000000001",
- "1c2c6512ff5c6a5fc000000000000000"),
- /* [ 63] */ new RandomConstant("af85d0c629e6c7de0000000000000001",
- "7858ca25feb8d4bf8000000000000000"),
- /* [ 64] */ new RandomConstant("5f0ba18c53cd8fbc0000000000000001",
- "f0b1944bfd71a97f0000000000000000"),
- /* [ 65] */ new RandomConstant("be174318a79b1f780000000000000001",
- "e1632897fae352fe0000000000000000"),
- /* [ 66] */ new RandomConstant("7c2e86314f363ef00000000000000001",
- "c2c6512ff5c6a5fc0000000000000000"),
- /* [ 67] */ new RandomConstant("f85d0c629e6c7de00000000000000001",
- "858ca25feb8d4bf80000000000000000"),
- /* [ 68] */ new RandomConstant("f0ba18c53cd8fbc00000000000000001",
- "0b1944bfd71a97f00000000000000000"),
- /* [ 69] */ new RandomConstant("e174318a79b1f7800000000000000001",
- "1632897fae352fe00000000000000000"),
- /* [ 70] */ new RandomConstant("c2e86314f363ef000000000000000001",
- "2c6512ff5c6a5fc00000000000000000"),
- /* [ 71] */ new RandomConstant("85d0c629e6c7de000000000000000001",
- "58ca25feb8d4bf800000000000000000"),
- /* [ 72] */ new RandomConstant("0ba18c53cd8fbc000000000000000001",
- "b1944bfd71a97f000000000000000000"),
- /* [ 73] */ new RandomConstant("174318a79b1f78000000000000000001",
- "632897fae352fe000000000000000000"),
- /* [ 74] */ new RandomConstant("2e86314f363ef0000000000000000001",
- "c6512ff5c6a5fc000000000000000000"),
- /* [ 75] */ new RandomConstant("5d0c629e6c7de0000000000000000001",
- "8ca25feb8d4bf8000000000000000000"),
- /* [ 76] */ new RandomConstant("ba18c53cd8fbc0000000000000000001",
- "1944bfd71a97f0000000000000000000"),
- /* [ 77] */ new RandomConstant("74318a79b1f780000000000000000001",
- "32897fae352fe0000000000000000000"),
- /* [ 78] */ new RandomConstant("e86314f363ef00000000000000000001",
- "6512ff5c6a5fc0000000000000000000"),
- /* [ 79] */ new RandomConstant("d0c629e6c7de00000000000000000001",
- "ca25feb8d4bf80000000000000000000"),
- /* [ 80] */ new RandomConstant("a18c53cd8fbc00000000000000000001",
- "944bfd71a97f00000000000000000000"),
- /* [ 81] */ new RandomConstant("4318a79b1f7800000000000000000001",
- "2897fae352fe00000000000000000000"),
- /* [ 82] */ new RandomConstant("86314f363ef000000000000000000001",
- "512ff5c6a5fc00000000000000000000"),
- /* [ 83] */ new RandomConstant("0c629e6c7de000000000000000000001",
- "a25feb8d4bf800000000000000000000"),
- /* [ 84] */ new RandomConstant("18c53cd8fbc000000000000000000001",
- "44bfd71a97f000000000000000000000"),
- /* [ 85] */ new RandomConstant("318a79b1f78000000000000000000001",
- "897fae352fe000000000000000000000"),
- /* [ 86] */ new RandomConstant("6314f363ef0000000000000000000001",
- "12ff5c6a5fc000000000000000000000"),
- /* [ 87] */ new RandomConstant("c629e6c7de0000000000000000000001",
- "25feb8d4bf8000000000000000000000"),
- /* [ 88] */ new RandomConstant("8c53cd8fbc0000000000000000000001",
- "4bfd71a97f0000000000000000000000"),
- /* [ 89] */ new RandomConstant("18a79b1f780000000000000000000001",
- "97fae352fe0000000000000000000000"),
- /* [ 90] */ new RandomConstant("314f363ef00000000000000000000001",
- "2ff5c6a5fc0000000000000000000000"),
- /* [ 91] */ new RandomConstant("629e6c7de00000000000000000000001",
- "5feb8d4bf80000000000000000000000"),
- /* [ 92] */ new RandomConstant("c53cd8fbc00000000000000000000001",
- "bfd71a97f00000000000000000000000"),
- /* [ 93] */ new RandomConstant("8a79b1f7800000000000000000000001",
- "7fae352fe00000000000000000000000"),
- /* [ 94] */ new RandomConstant("14f363ef000000000000000000000001",
- "ff5c6a5fc00000000000000000000000"),
- /* [ 95] */ new RandomConstant("29e6c7de000000000000000000000001",
- "feb8d4bf800000000000000000000000"),
- /* [ 96] */ new RandomConstant("53cd8fbc000000000000000000000001",
- "fd71a97f000000000000000000000000"),
- /* [ 97] */ new RandomConstant("a79b1f78000000000000000000000001",
- "fae352fe000000000000000000000000"),
- /* [ 98] */ new RandomConstant("4f363ef0000000000000000000000001",
- "f5c6a5fc000000000000000000000000"),
- /* [ 99] */ new RandomConstant("9e6c7de0000000000000000000000001",
- "eb8d4bf8000000000000000000000000"),
- /* [100] */ new RandomConstant("3cd8fbc0000000000000000000000001",
- "d71a97f0000000000000000000000000"),
- /* [101] */ new RandomConstant("79b1f780000000000000000000000001",
- "ae352fe0000000000000000000000000"),
- /* [102] */ new RandomConstant("f363ef00000000000000000000000001",
- "5c6a5fc0000000000000000000000000"),
- /* [103] */ new RandomConstant("e6c7de00000000000000000000000001",
- "b8d4bf80000000000000000000000000"),
- /* [104] */ new RandomConstant("cd8fbc00000000000000000000000001",
- "71a97f00000000000000000000000000"),
- /* [105] */ new RandomConstant("9b1f7800000000000000000000000001",
- "e352fe00000000000000000000000000"),
- /* [106] */ new RandomConstant("363ef000000000000000000000000001",
- "c6a5fc00000000000000000000000000"),
- /* [107] */ new RandomConstant("6c7de000000000000000000000000001",
- "8d4bf800000000000000000000000000"),
- /* [108] */ new RandomConstant("d8fbc000000000000000000000000001",
- "1a97f000000000000000000000000000"),
- /* [109] */ new RandomConstant("b1f78000000000000000000000000001",
- "352fe000000000000000000000000000"),
- /* [110] */ new RandomConstant("63ef0000000000000000000000000001",
- "6a5fc000000000000000000000000000"),
- /* [111] */ new RandomConstant("c7de0000000000000000000000000001",
- "d4bf8000000000000000000000000000"),
- /* [112] */ new RandomConstant("8fbc0000000000000000000000000001",
- "a97f0000000000000000000000000000"),
- /* [113] */ new RandomConstant("1f780000000000000000000000000001",
- "52fe0000000000000000000000000000"),
- /* [114] */ new RandomConstant("3ef00000000000000000000000000001",
- "a5fc0000000000000000000000000000"),
- /* [115] */ new RandomConstant("7de00000000000000000000000000001",
- "4bf80000000000000000000000000000"),
- /* [116] */ new RandomConstant("fbc00000000000000000000000000001",
- "97f00000000000000000000000000000"),
- /* [117] */ new RandomConstant("f7800000000000000000000000000001",
- "2fe00000000000000000000000000000"),
- /* [118] */ new RandomConstant("ef000000000000000000000000000001",
- "5fc00000000000000000000000000000"),
- /* [119] */ new RandomConstant("de000000000000000000000000000001",
- "bf800000000000000000000000000000"),
- /* [120] */ new RandomConstant("bc000000000000000000000000000001",
- "7f000000000000000000000000000000"),
- /* [121] */ new RandomConstant("78000000000000000000000000000001",
- "fe000000000000000000000000000000"),
- /* [122] */ new RandomConstant("f0000000000000000000000000000001",
- "fc000000000000000000000000000000"),
- /* [123] */ new RandomConstant("e0000000000000000000000000000001",
- "f8000000000000000000000000000000"),
- /* [124] */ new RandomConstant("c0000000000000000000000000000001",
- "f0000000000000000000000000000000"),
- /* [125] */ new RandomConstant("80000000000000000000000000000001",
- "e0000000000000000000000000000000"),
- /* [126] */ new RandomConstant("00000000000000000000000000000001",
- "c0000000000000000000000000000000"),
- /* [127] */ new RandomConstant("00000000000000000000000000000001",
- "80000000000000000000000000000000")};
-
- /**
- * generate the random number that is "advance" steps
- * from an initial random number of 0. This is done by
- * starting with 0, and then advancing the by the
- * appropriate powers of 2 of the linear congruential
- * generator.
- */
- public static Unsigned16 skipAhead(Unsigned16 advance) {
- Unsigned16 result = new Unsigned16();
- long bit_map;
-
- bit_map = advance.getLow8();
- for (int i = 0; bit_map != 0 && i < 64; i++) {
- if ((bit_map & (1L << i)) != 0) {
- /* advance random number by f**(2**i) (x)
- */
- result.multiply(genArray[i].a);
- result.add(genArray[i].c);
- bit_map &= ~(1L << i);
- }
- }
- bit_map = advance.getHigh8();
- for (int i = 0; bit_map != 0 && i < 64; i++)
- {
- if ((bit_map & (1L << i)) != 0) {
- /* advance random number by f**(2**(i + 64)) (x)
- */
- result.multiply(genArray[i+64].a);
- result.add(genArray[i+64].c);
- bit_map &= ~(1L << i);
- }
- }
- return result;
- }
-
- /**
- * Generate the next 16 byte random number.
- */
- public static void nextRand(Unsigned16 rand) {
- /* advance the random number forward once using the linear congruential
- * generator, and then return the new random number
- */
- rand.multiply(genArray[0].a);
- rand.add(genArray[0].c);
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraChecksum.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraChecksum.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraChecksum.java
deleted file mode 100644
index a5b408b..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraChecksum.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tez.mapreduce.examples.terasort;
-
-import java.io.IOException;
-import java.util.zip.Checksum;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.util.PureJavaCrc32;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-public class TeraChecksum extends Configured implements Tool {
- static class ChecksumMapper
- extends Mapper<Text, Text, NullWritable, Unsigned16> {
- private Unsigned16 checksum = new Unsigned16();
- private Unsigned16 sum = new Unsigned16();
- private Checksum crc32 = new PureJavaCrc32();
-
- public void map(Text key, Text value,
- Context context) throws IOException {
- crc32.reset();
- crc32.update(key.getBytes(), 0, key.getLength());
- crc32.update(value.getBytes(), 0, value.getLength());
- checksum.set(crc32.getValue());
- sum.add(checksum);
- }
-
- public void cleanup(Context context)
- throws IOException, InterruptedException {
- context.write(NullWritable.get(), sum);
- }
- }
-
- static class ChecksumReducer
- extends Reducer<NullWritable, Unsigned16, NullWritable, Unsigned16> {
-
- public void reduce(NullWritable key, Iterable<Unsigned16> values,
- Context context) throws IOException, InterruptedException {
- Unsigned16 sum = new Unsigned16();
- for (Unsigned16 val : values) {
- sum.add(val);
- }
- context.write(key, sum);
- }
- }
-
- private static void usage() throws IOException {
- System.err.println("terasum <out-dir> <report-dir>");
- }
-
- public int run(String[] args) throws Exception {
- Job job = Job.getInstance(getConf());
- if (args.length != 2) {
- usage();
- return 2;
- }
- TeraInputFormat.setInputPaths(job, new Path(args[0]));
- FileOutputFormat.setOutputPath(job, new Path(args[1]));
- job.setJobName("TeraSum");
- job.setJarByClass(TeraChecksum.class);
- job.setMapperClass(ChecksumMapper.class);
- job.setReducerClass(ChecksumReducer.class);
- job.setOutputKeyClass(NullWritable.class);
- job.setOutputValueClass(Unsigned16.class);
- // force a single reducer
- job.setNumReduceTasks(1);
- job.setInputFormatClass(TeraInputFormat.class);
- return job.waitForCompletion(true) ? 0 : 1;
- }
-
- /**
- * @param args
- */
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new TeraChecksum(), args);
- System.exit(res);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraGen.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraGen.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraGen.java
deleted file mode 100644
index 70a4207..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraGen.java
+++ /dev/null
@@ -1,311 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples.terasort;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.zip.Checksum;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableUtils;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.util.PureJavaCrc32;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-/**
- * Generate the official GraySort input data set.
- * The user specifies the number of rows and the output directory and this
- * class runs a map/reduce program to generate the data.
- * The format of the data is:
- * <ul>
- * <li>(10 bytes key) (constant 2 bytes) (32 bytes rowid)
- * (constant 4 bytes) (48 bytes filler) (constant 4 bytes)
- * <li>The rowid is the right justified row id as a hex number.
- * </ul>
- *
- * <p>
- * To run the program:
- * <b>bin/hadoop jar hadoop-*-examples.jar teragen 10000000000 in-dir</b>
- */
-public class TeraGen extends Configured implements Tool {
- private static final Log LOG = LogFactory.getLog(TeraSort.class);
-
- public static enum Counters {CHECKSUM}
-
- public static final String NUM_ROWS = "mapreduce.terasort.num-rows";
- /**
- * An input format that assigns ranges of longs to each mapper.
- */
- static class RangeInputFormat
- extends InputFormat<LongWritable, NullWritable> {
-
- /**
- * An input split consisting of a range on numbers.
- */
- static class RangeInputSplit extends InputSplit implements Writable {
- long firstRow;
- long rowCount;
-
- public RangeInputSplit() { }
-
- public RangeInputSplit(long offset, long length) {
- firstRow = offset;
- rowCount = length;
- }
-
- public long getLength() throws IOException {
- return 0;
- }
-
- public String[] getLocations() throws IOException {
- return new String[]{};
- }
-
- public void readFields(DataInput in) throws IOException {
- firstRow = WritableUtils.readVLong(in);
- rowCount = WritableUtils.readVLong(in);
- }
-
- public void write(DataOutput out) throws IOException {
- WritableUtils.writeVLong(out, firstRow);
- WritableUtils.writeVLong(out, rowCount);
- }
- }
-
- /**
- * A record reader that will generate a range of numbers.
- */
- static class RangeRecordReader
- extends RecordReader<LongWritable, NullWritable> {
- long startRow;
- long finishedRows;
- long totalRows;
- LongWritable key = null;
-
- public RangeRecordReader() {
- }
-
- public void initialize(InputSplit split, TaskAttemptContext context)
- throws IOException, InterruptedException {
- startRow = ((RangeInputSplit)split).firstRow;
- finishedRows = 0;
- totalRows = ((RangeInputSplit)split).rowCount;
- }
-
- public void close() throws IOException {
- // NOTHING
- }
-
- public LongWritable getCurrentKey() {
- return key;
- }
-
- public NullWritable getCurrentValue() {
- return NullWritable.get();
- }
-
- public float getProgress() throws IOException {
- return finishedRows / (float) totalRows;
- }
-
- public boolean nextKeyValue() {
- if (key == null) {
- key = new LongWritable();
- }
- if (finishedRows < totalRows) {
- key.set(startRow + finishedRows);
- finishedRows += 1;
- return true;
- } else {
- return false;
- }
- }
-
- }
-
- public RecordReader<LongWritable, NullWritable>
- createRecordReader(InputSplit split, TaskAttemptContext context)
- throws IOException {
- return new RangeRecordReader();
- }
-
- /**
- * Create the desired number of splits, dividing the number of rows
- * between the mappers.
- */
- public List<InputSplit> getSplits(JobContext job) {
- long totalRows = getNumberOfRows(job);
- int numSplits = job.getConfiguration().getInt(MRJobConfig.NUM_MAPS, 1);
- LOG.info("Generating " + totalRows + " using " + numSplits);
- List<InputSplit> splits = new ArrayList<InputSplit>();
- long currentRow = 0;
- for(int split = 0; split < numSplits; ++split) {
- long goal =
- (long) Math.ceil(totalRows * (double)(split + 1) / numSplits);
- splits.add(new RangeInputSplit(currentRow, goal - currentRow));
- currentRow = goal;
- }
- return splits;
- }
-
- }
-
- static long getNumberOfRows(JobContext job) {
- return job.getConfiguration().getLong(NUM_ROWS, 0);
- }
-
- static void setNumberOfRows(Job job, long numRows) {
- job.getConfiguration().setLong(NUM_ROWS, numRows);
- }
-
- /**
- * The Mapper class that given a row number, will generate the appropriate
- * output line.
- */
- public static class SortGenMapper
- extends Mapper<LongWritable, NullWritable, Text, Text> {
-
- private Text key = new Text();
- private Text value = new Text();
- private Unsigned16 rand = null;
- private Unsigned16 rowId = null;
- private Unsigned16 checksum = new Unsigned16();
- private Checksum crc32 = new PureJavaCrc32();
- private Unsigned16 total = new Unsigned16();
- private static final Unsigned16 ONE = new Unsigned16(1);
- private byte[] buffer = new byte[TeraInputFormat.KEY_LENGTH +
- TeraInputFormat.VALUE_LENGTH];
- private Counter checksumCounter;
-
- public void map(LongWritable row, NullWritable ignored,
- Context context) throws IOException, InterruptedException {
- if (rand == null) {
- rowId = new Unsigned16(row.get());
- rand = Random16.skipAhead(rowId);
- checksumCounter = context.getCounter(Counters.CHECKSUM);
- }
- Random16.nextRand(rand);
- GenSort.generateRecord(buffer, rand, rowId);
- key.set(buffer, 0, TeraInputFormat.KEY_LENGTH);
- value.set(buffer, TeraInputFormat.KEY_LENGTH,
- TeraInputFormat.VALUE_LENGTH);
- context.write(key, value);
- crc32.reset();
- crc32.update(buffer, 0,
- TeraInputFormat.KEY_LENGTH + TeraInputFormat.VALUE_LENGTH);
- checksum.set(crc32.getValue());
- total.add(checksum);
- rowId.add(ONE);
- }
-
- @Override
- public void cleanup(Context context) {
- if (checksumCounter != null) {
- checksumCounter.increment(total.getLow8());
- }
- }
- }
-
- private static void usage() throws IOException {
- System.err.println("teragen <num rows> <output dir>");
- }
-
- /**
- * Parse a number that optionally has a postfix that denotes a base.
- * @param str an string integer with an option base {k,m,b,t}.
- * @return the expanded value
- */
- private static long parseHumanLong(String str) {
- char tail = str.charAt(str.length() - 1);
- long base = 1;
- switch (tail) {
- case 't':
- base *= 1000 * 1000 * 1000 * 1000;
- break;
- case 'b':
- base *= 1000 * 1000 * 1000;
- break;
- case 'm':
- base *= 1000 * 1000;
- break;
- case 'k':
- base *= 1000;
- break;
- default:
- }
- if (base != 1) {
- str = str.substring(0, str.length() - 1);
- }
- return Long.parseLong(str) * base;
- }
-
- /**
- * @param args the cli arguments
- */
- public int run(String[] args)
- throws IOException, InterruptedException, ClassNotFoundException {
- Job job = Job.getInstance(getConf());
- if (args.length != 2) {
- usage();
- return 2;
- }
- setNumberOfRows(job, parseHumanLong(args[0]));
- Path outputDir = new Path(args[1]);
- if (outputDir.getFileSystem(getConf()).exists(outputDir)) {
- throw new IOException("Output directory " + outputDir +
- " already exists.");
- }
- FileOutputFormat.setOutputPath(job, outputDir);
- job.setJobName("TeraGen");
- job.setJarByClass(TeraGen.class);
- job.setMapperClass(SortGenMapper.class);
- job.setNumReduceTasks(0);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(Text.class);
- job.setInputFormatClass(RangeInputFormat.class);
- job.setOutputFormatClass(TeraOutputFormat.class);
- return job.waitForCompletion(true) ? 0 : 1;
- }
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new TeraGen(), args);
- System.exit(res);
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraInputFormat.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraInputFormat.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraInputFormat.java
deleted file mode 100644
index 94ee340..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraInputFormat.java
+++ /dev/null
@@ -1,353 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples.terasort;
-
-import java.io.DataOutputStream;
-import java.io.EOFException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
-import org.apache.hadoop.util.IndexedSortable;
-import org.apache.hadoop.util.QuickSort;
-import org.apache.hadoop.util.StringUtils;
-
-/**
- * An input format that reads the first 10 characters of each line as the key
- * and the rest of the line as the value. Both key and value are represented
- * as Text.
- */
-public class TeraInputFormat extends FileInputFormat<Text,Text> {
-
- static final String PARTITION_FILENAME = "_partition.lst";
- private static final String NUM_PARTITIONS =
- "mapreduce.terasort.num.partitions";
- private static final String SAMPLE_SIZE =
- "mapreduce.terasort.partitions.sample";
- static final int KEY_LENGTH = 10;
- static final int VALUE_LENGTH = 90;
- static final int RECORD_LENGTH = KEY_LENGTH + VALUE_LENGTH;
- private static MRJobConfig lastContext = null;
- private static List<InputSplit> lastResult = null;
-
- static class TeraFileSplit extends FileSplit {
- static private String[] ZERO_LOCATIONS = new String[0];
-
- private String[] locations;
-
- public TeraFileSplit() {
- locations = ZERO_LOCATIONS;
- }
- public TeraFileSplit(Path file, long start, long length, String[] hosts) {
- super(file, start, length, hosts);
- try {
- locations = super.getLocations();
- } catch (IOException e) {
- locations = ZERO_LOCATIONS;
- }
- }
-
- // XXXXXX should this also be null-protected?
- protected void setLocations(String[] hosts) {
- locations = hosts;
- }
-
- @Override
- public String[] getLocations() {
- return locations;
- }
-
- public String toString() {
- StringBuffer result = new StringBuffer();
- result.append(getPath());
- result.append(" from ");
- result.append(getStart());
- result.append(" length ");
- result.append(getLength());
- for(String host: getLocations()) {
- result.append(" ");
- result.append(host);
- }
- return result.toString();
- }
- }
-
- static class TextSampler implements IndexedSortable {
- private ArrayList<Text> records = new ArrayList<Text>();
-
- public int compare(int i, int j) {
- Text left = records.get(i);
- Text right = records.get(j);
- return left.compareTo(right);
- }
-
- public void swap(int i, int j) {
- Text left = records.get(i);
- Text right = records.get(j);
- records.set(j, left);
- records.set(i, right);
- }
-
- public void addKey(Text key) {
- synchronized (this) {
- records.add(new Text(key));
- }
- }
-
- /**
- * Find the split points for a given sample. The sample keys are sorted
- * and down sampled to find even split points for the partitions. The
- * returned keys should be the start of their respective partitions.
- * @param numPartitions the desired number of partitions
- * @return an array of size numPartitions - 1 that holds the split points
- */
- Text[] createPartitions(int numPartitions) {
- int numRecords = records.size();
- System.out.println("Making " + numPartitions + " from " + numRecords +
- " sampled records");
- if (numPartitions > numRecords) {
- throw new IllegalArgumentException
- ("Requested more partitions than input keys (" + numPartitions +
- " > " + numRecords + ")");
- }
- new QuickSort().sort(this, 0, records.size());
- float stepSize = numRecords / (float) numPartitions;
- Text[] result = new Text[numPartitions-1];
- for(int i=1; i < numPartitions; ++i) {
- result[i-1] = records.get(Math.round(stepSize * i));
- }
- return result;
- }
- }
-
- /**
- * Use the input splits to take samples of the input and generate sample
- * keys. By default reads 100,000 keys from 10 locations in the input, sorts
- * them and picks N-1 keys to generate N equally sized partitions.
- * @param job the job to sample
- * @param partFile where to write the output file to
- * @throws Throwable if something goes wrong
- */
- public static void writePartitionFile(final JobContext job,
- Path partFile) throws Throwable {
- long t1 = System.currentTimeMillis();
- Configuration conf = job.getConfiguration();
- final TeraInputFormat inFormat = new TeraInputFormat();
- final TextSampler sampler = new TextSampler();
- int partitions = job.getNumReduceTasks();
- long sampleSize = conf.getLong(SAMPLE_SIZE, 100000);
- final List<InputSplit> splits = inFormat.getSplits(job);
- long t2 = System.currentTimeMillis();
- System.out.println("Computing input splits took " + (t2 - t1) + "ms");
- int samples = Math.min(conf.getInt(NUM_PARTITIONS, 10), splits.size());
- System.out.println("Sampling " + samples + " splits of " + splits.size());
- final long recordsPerSample = sampleSize / samples;
- final int sampleStep = splits.size() / samples;
- Thread[] samplerReader = new Thread[samples];
- SamplerThreadGroup threadGroup = new SamplerThreadGroup("Sampler Reader Thread Group");
- // take N samples from different parts of the input
- for(int i=0; i < samples; ++i) {
- final int idx = i;
- samplerReader[i] =
- new Thread (threadGroup,"Sampler Reader " + idx) {
- {
- setDaemon(true);
- }
- public void run() {
- long records = 0;
- try {
- TaskAttemptContext context = new TaskAttemptContextImpl(
- job.getConfiguration(), new TaskAttemptID());
- RecordReader<Text, Text> reader =
- inFormat.createRecordReader(splits.get(sampleStep * idx),
- context);
- reader.initialize(splits.get(sampleStep * idx), context);
- while (reader.nextKeyValue()) {
- sampler.addKey(new Text(reader.getCurrentKey()));
- records += 1;
- if (recordsPerSample <= records) {
- break;
- }
- }
- } catch (IOException ie){
- System.err.println("Got an exception while reading splits " +
- StringUtils.stringifyException(ie));
- throw new RuntimeException(ie);
- } catch (InterruptedException e) {
-
- }
- }
- };
- samplerReader[i].start();
- }
- FileSystem outFs = partFile.getFileSystem(conf);
- DataOutputStream writer = outFs.create(partFile, true, 64*1024, (short) 10,
- outFs.getDefaultBlockSize(partFile));
- for (int i = 0; i < samples; i++) {
- try {
- samplerReader[i].join();
- if(threadGroup.getThrowable() != null){
- throw threadGroup.getThrowable();
- }
- } catch (InterruptedException e) {
- }
- }
- for(Text split : sampler.createPartitions(partitions)) {
- split.write(writer);
- }
- writer.close();
- long t3 = System.currentTimeMillis();
- System.out.println("Computing parititions took " + (t3 - t2) + "ms");
- }
-
- static class SamplerThreadGroup extends ThreadGroup{
-
- private Throwable throwable;
-
- public SamplerThreadGroup(String s) {
- super(s);
- }
-
- @Override
- public void uncaughtException(Thread thread, Throwable throwable) {
- this.throwable = throwable;
- }
-
- public Throwable getThrowable() {
- return this.throwable;
- }
-
- }
-
- static class TeraRecordReader extends RecordReader<Text,Text> {
- private FSDataInputStream in;
- private long offset;
- private long length;
- private static final int RECORD_LENGTH = KEY_LENGTH + VALUE_LENGTH;
- private byte[] buffer = new byte[RECORD_LENGTH];
- private Text key;
- private Text value;
-
- public TeraRecordReader() throws IOException {
- }
-
- public void initialize(InputSplit split, TaskAttemptContext context)
- throws IOException, InterruptedException {
- Path p = ((FileSplit)split).getPath();
- FileSystem fs = p.getFileSystem(context.getConfiguration());
- in = fs.open(p);
- long start = ((FileSplit)split).getStart();
- // find the offset to start at a record boundary
- offset = (RECORD_LENGTH - (start % RECORD_LENGTH)) % RECORD_LENGTH;
- in.seek(start + offset);
- length = ((FileSplit)split).getLength();
- }
-
- public void close() throws IOException {
- in.close();
- }
-
- public Text getCurrentKey() {
- return key;
- }
-
- public Text getCurrentValue() {
- return value;
- }
-
- public float getProgress() throws IOException {
- return (float) offset / length;
- }
-
- public boolean nextKeyValue() throws IOException {
- if (offset >= length) {
- return false;
- }
- int read = 0;
- while (read < RECORD_LENGTH) {
- long newRead = in.read(buffer, read, RECORD_LENGTH - read);
- if (newRead == -1) {
- if (read == 0) {
- return false;
- } else {
- throw new EOFException("read past eof");
- }
- }
- read += newRead;
- }
- if (key == null) {
- key = new Text();
- }
- if (value == null) {
- value = new Text();
- }
- key.set(buffer, 0, KEY_LENGTH);
- value.set(buffer, KEY_LENGTH, VALUE_LENGTH);
- offset += RECORD_LENGTH;
- return true;
- }
- }
-
- @Override
- public RecordReader<Text, Text>
- createRecordReader(InputSplit split, TaskAttemptContext context)
- throws IOException {
- return new TeraRecordReader();
- }
-
- protected FileSplit makeSplit(Path file, long start, long length,
- String[] hosts) {
- return new TeraFileSplit(file, start, length, hosts);
- }
-
- @Override
- public List<InputSplit> getSplits(JobContext job) throws IOException {
- if (job == lastContext) {
- return lastResult;
- }
- long t1, t2, t3;
- t1 = System.currentTimeMillis();
- lastContext = job;
- lastResult = super.getSplits(job);
- t2 = System.currentTimeMillis();
- System.out.println("Spent " + (t2 - t1) + "ms computing base-splits.");
- if (job.getConfiguration().getBoolean(TeraScheduler.USE, true)) {
- TeraScheduler scheduler = new TeraScheduler(
- lastResult.toArray(new TeraFileSplit[0]), job.getConfiguration());
- lastResult = scheduler.getNewFileSplits();
- t3 = System.currentTimeMillis();
- System.out.println("Spent " + (t3 - t2) + "ms computing TeraScheduler splits.");
- }
- return lastResult;
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraOutputFormat.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraOutputFormat.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraOutputFormat.java
deleted file mode 100644
index 4ff0f06..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraOutputFormat.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples.terasort;
-
-import java.io.IOException;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.InvalidJobConfException;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.security.TokenCache;
-
-/**
- * An output format that writes the key and value appended together.
- */
-public class TeraOutputFormat extends FileOutputFormat<Text,Text> {
- static final String FINAL_SYNC_ATTRIBUTE = "mapreduce.terasort.final.sync";
- private OutputCommitter committer = null;
-
- /**
- * Set the requirement for a final sync before the stream is closed.
- */
- static void setFinalSync(JobContext job, boolean newValue) {
- job.getConfiguration().setBoolean(FINAL_SYNC_ATTRIBUTE, newValue);
- }
-
- /**
- * Does the user want a final sync at close?
- */
- public static boolean getFinalSync(JobContext job) {
- return job.getConfiguration().getBoolean(FINAL_SYNC_ATTRIBUTE, false);
- }
-
- static class TeraRecordWriter extends RecordWriter<Text,Text> {
- private boolean finalSync = false;
- private FSDataOutputStream out;
-
- public TeraRecordWriter(FSDataOutputStream out,
- JobContext job) {
- finalSync = getFinalSync(job);
- this.out = out;
- }
-
- public synchronized void write(Text key,
- Text value) throws IOException {
- out.write(key.getBytes(), 0, key.getLength());
- out.write(value.getBytes(), 0, value.getLength());
- }
-
- public void close(TaskAttemptContext context) throws IOException {
- if (finalSync) {
- out.hsync();
- }
- out.close();
- }
- }
-
- @Override
- public void checkOutputSpecs(JobContext job
- ) throws InvalidJobConfException, IOException {
- // Ensure that the output directory is set
- Path outDir = getOutputPath(job);
- if (outDir == null) {
- throw new InvalidJobConfException("Output directory not set in JobConf.");
- }
-
- // get delegation token for outDir's file system
- TokenCache.obtainTokensForNamenodes(job.getCredentials(),
- new Path[] { outDir }, job.getConfiguration());
- }
-
- public RecordWriter<Text,Text> getRecordWriter(TaskAttemptContext job
- ) throws IOException {
- Path file = getDefaultWorkFile(job, "");
- FileSystem fs = file.getFileSystem(job.getConfiguration());
- FSDataOutputStream fileOut = fs.create(file);
- return new TeraRecordWriter(fileOut, job);
- }
-
- public OutputCommitter getOutputCommitter(TaskAttemptContext context)
- throws IOException {
- if (committer == null) {
- Path output = getOutputPath(context);
- committer = new FileOutputCommitter(output, context);
- }
- return committer;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraScheduler.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraScheduler.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraScheduler.java
deleted file mode 100644
index 82638d8..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraScheduler.java
+++ /dev/null
@@ -1,253 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples.terasort;
-
-import java.io.*;
-import java.util.*;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
-import org.apache.tez.mapreduce.examples.terasort.TeraInputFormat.TeraFileSplit;
-
-import com.google.common.base.Charsets;
-
-class TeraScheduler {
- static String USE = "mapreduce.terasort.use.terascheduler";
- private static final Log LOG = LogFactory.getLog(TeraScheduler.class);
- private Split[] splits;
- private List<Host> hosts = new ArrayList<Host>();
- private int slotsPerHost;
- private int remainingSplits = 0;
- private FileSplit[] realSplits = null;
-
- static class Split {
- String filename;
- boolean isAssigned = false;
- List<Host> locations = new ArrayList<Host>();
- Split(String filename) {
- this.filename = filename;
- }
- public String toString() {
- StringBuffer result = new StringBuffer();
- result.append(filename);
- result.append(" on ");
- for(Host host: locations) {
- result.append(host.hostname);
- result.append(", ");
- }
- return result.toString();
- }
- }
- static class Host {
- String hostname;
- List<Split> splits = new ArrayList<Split>();
- Host(String hostname) {
- this.hostname = hostname;
- }
- public String toString() {
- StringBuffer result = new StringBuffer();
- result.append(splits.size());
- result.append(" ");
- result.append(hostname);
- return result.toString();
- }
- }
-
- List<String> readFile(String filename) throws IOException {
- List<String> result = new ArrayList<String>(10000);
- BufferedReader in = new BufferedReader(
- new InputStreamReader(new FileInputStream(filename), Charsets.UTF_8));
- String line = in.readLine();
- while (line != null) {
- result.add(line);
- line = in.readLine();
- }
- in.close();
- return result;
- }
-
- public TeraScheduler(String splitFilename,
- String nodeFilename) throws IOException {
- slotsPerHost = 4;
- // get the hosts
- Map<String, Host> hostIds = new HashMap<String,Host>();
- for(String hostName: readFile(nodeFilename)) {
- Host host = new Host(hostName);
- hosts.add(host);
- hostIds.put(hostName, host);
- }
- // read the blocks
- List<String> splitLines = readFile(splitFilename);
- splits = new Split[splitLines.size()];
- remainingSplits = 0;
- for(String line: splitLines) {
- StringTokenizer itr = new StringTokenizer(line);
- Split newSplit = new Split(itr.nextToken());
- splits[remainingSplits++] = newSplit;
- while (itr.hasMoreTokens()) {
- Host host = hostIds.get(itr.nextToken());
- newSplit.locations.add(host);
- host.splits.add(newSplit);
- }
- }
- }
-
- public TeraScheduler(FileSplit[] realSplits,
- Configuration conf) throws IOException {
- this.realSplits = realSplits;
- this.slotsPerHost = conf.getInt(TTConfig.TT_MAP_SLOTS, 4);
- Map<String, Host> hostTable = new HashMap<String, Host>();
- splits = new Split[realSplits.length];
- for(FileSplit realSplit: realSplits) {
- Split split = new Split(realSplit.getPath().toString());
- splits[remainingSplits++] = split;
- for(String hostname: realSplit.getLocations()) {
- Host host = hostTable.get(hostname);
- if (host == null) {
- host = new Host(hostname);
- hostTable.put(hostname, host);
- hosts.add(host);
- }
- host.splits.add(split);
- split.locations.add(host);
- }
- }
- }
-
- Host pickBestHost() {
- Host result = null;
- int splits = Integer.MAX_VALUE;
- for(Host host: hosts) {
- if (host.splits.size() < splits) {
- result = host;
- splits = host.splits.size();
- }
- }
- if (result != null) {
- hosts.remove(result);
- LOG.debug("picking " + result);
- }
- return result;
- }
-
- void pickBestSplits(Host host) {
- int tasksToPick = Math.min(slotsPerHost,
- (int) Math.ceil((double) remainingSplits /
- hosts.size()));
- Split[] best = new Split[tasksToPick];
- for(Split cur: host.splits) {
- LOG.debug(" examine: " + cur.filename + " " + cur.locations.size());
- int i = 0;
- while (i < tasksToPick && best[i] != null &&
- best[i].locations.size() <= cur.locations.size()) {
- i += 1;
- }
- if (i < tasksToPick) {
- for(int j = tasksToPick - 1; j > i; --j) {
- best[j] = best[j-1];
- }
- best[i] = cur;
- }
- }
- // for the chosen blocks, remove them from the other locations
- for(int i=0; i < tasksToPick; ++i) {
- if (best[i] != null) {
- LOG.debug(" best: " + best[i].filename);
- for (Host other: best[i].locations) {
- other.splits.remove(best[i]);
- }
- best[i].locations.clear();
- best[i].locations.add(host);
- best[i].isAssigned = true;
- remainingSplits -= 1;
- }
- }
- // for the non-chosen blocks, remove this host
- for(Split cur: host.splits) {
- if (!cur.isAssigned) {
- cur.locations.remove(host);
- }
- }
- }
-
- void solve() throws IOException {
- Host host = pickBestHost();
- while (host != null) {
- pickBestSplits(host);
- host = pickBestHost();
- }
- }
-
- /**
- * Solve the schedule and modify the FileSplit array to reflect the new
- * schedule. It will move placed splits to front and unplacable splits
- * to the end.
- * @return a new list of FileSplits that are modified to have the
- * best host as the only host.
- * @throws IOException
- */
- public List<InputSplit> getNewFileSplits() throws IOException {
- solve();
- FileSplit[] result = new FileSplit[realSplits.length];
- int left = 0;
- int right = realSplits.length - 1;
- for(int i=0; i < splits.length; ++i) {
- if (splits[i].isAssigned) {
- // copy the split and fix up the locations
- ((TeraFileSplit) realSplits[i]).setLocations
- (new String[]{splits[i].locations.get(0).hostname});
- result[left++] = realSplits[i];
- } else {
- result[right--] = realSplits[i];
- }
- }
- List<InputSplit> ret = new ArrayList<InputSplit>();
- for (FileSplit fs : result) {
- ret.add(fs);
- }
- return ret;
- }
-
- public static void main(String[] args) throws IOException {
- TeraScheduler problem = new TeraScheduler("block-loc.txt", "nodes");
- for(Host host: problem.hosts) {
- System.out.println(host);
- }
- LOG.info("starting solve");
- problem.solve();
- List<Split> leftOvers = new ArrayList<Split>();
- for(int i=0; i < problem.splits.length; ++i) {
- if (problem.splits[i].isAssigned) {
- System.out.println("sched: " + problem.splits[i]);
- } else {
- leftOvers.add(problem.splits[i]);
- }
- }
- for(Split cur: leftOvers) {
- System.out.println("left: " + cur);
- }
- System.out.println("left over: " + leftOvers.size());
- LOG.info("done");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraSort.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraSort.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraSort.java
deleted file mode 100644
index 5a097f2..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraSort.java
+++ /dev/null
@@ -1,335 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples.terasort;
-
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.net.URI;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.Partitioner;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-/**
- * Generates the sampled split points, launches the job, and waits for it to
- * finish.
- * <p>
- * To run the program:
- * <b>bin/hadoop jar hadoop-*-examples.jar terasort in-dir out-dir</b>
- */
-public class TeraSort extends Configured implements Tool {
- private static final Log LOG = LogFactory.getLog(TeraSort.class);
- static String SIMPLE_PARTITIONER = "mapreduce.terasort.simplepartitioner";
- static String OUTPUT_REPLICATION = "mapreduce.terasort.output.replication";
-
- /**
- * A partitioner that splits text keys into roughly equal partitions
- * in a global sorted order.
- */
- static class TotalOrderPartitioner extends Partitioner<Text,Text>
- implements Configurable {
- private TrieNode trie;
- private Text[] splitPoints;
- private Configuration conf;
-
- /**
- * A generic trie node
- */
- static abstract class TrieNode {
- private int level;
- TrieNode(int level) {
- this.level = level;
- }
- abstract int findPartition(Text key);
- abstract void print(PrintStream strm) throws IOException;
- int getLevel() {
- return level;
- }
- }
-
- /**
- * An inner trie node that contains 256 children based on the next
- * character.
- */
- static class InnerTrieNode extends TrieNode {
- private TrieNode[] child = new TrieNode[256];
-
- InnerTrieNode(int level) {
- super(level);
- }
- int findPartition(Text key) {
- int level = getLevel();
- if (key.getLength() <= level) {
- return child[0].findPartition(key);
- }
- return child[key.getBytes()[level] & 0xff].findPartition(key);
- }
- void setChild(int idx, TrieNode child) {
- this.child[idx] = child;
- }
- void print(PrintStream strm) throws IOException {
- for(int ch=0; ch < 256; ++ch) {
- for(int i = 0; i < 2*getLevel(); ++i) {
- strm.print(' ');
- }
- strm.print(ch);
- strm.println(" ->");
- if (child[ch] != null) {
- child[ch].print(strm);
- }
- }
- }
- }
-
- /**
- * A leaf trie node that does string compares to figure out where the given
- * key belongs between lower..upper.
- */
- static class LeafTrieNode extends TrieNode {
- int lower;
- int upper;
- Text[] splitPoints;
- LeafTrieNode(int level, Text[] splitPoints, int lower, int upper) {
- super(level);
- this.splitPoints = splitPoints;
- this.lower = lower;
- this.upper = upper;
- }
- int findPartition(Text key) {
- for(int i=lower; i<upper; ++i) {
- if (splitPoints[i].compareTo(key) > 0) {
- return i;
- }
- }
- return upper;
- }
- void print(PrintStream strm) throws IOException {
- for(int i = 0; i < 2*getLevel(); ++i) {
- strm.print(' ');
- }
- strm.print(lower);
- strm.print(", ");
- strm.println(upper);
- }
- }
-
-
- /**
- * Read the cut points from the given sequence file.
- * @param fs the file system
- * @param p the path to read
- * @param job the job config
- * @return the strings to split the partitions on
- * @throws IOException
- */
- private static Text[] readPartitions(FileSystem fs, Path p,
- Configuration conf) throws IOException {
- int reduces = conf.getInt(MRJobConfig.NUM_REDUCES, 1);
- Text[] result = new Text[reduces - 1];
- DataInputStream reader = fs.open(p);
- for(int i=0; i < reduces - 1; ++i) {
- result[i] = new Text();
- result[i].readFields(reader);
- }
- reader.close();
- return result;
- }
-
- /**
- * Given a sorted set of cut points, build a trie that will find the correct
- * partition quickly.
- * @param splits the list of cut points
- * @param lower the lower bound of partitions 0..numPartitions-1
- * @param upper the upper bound of partitions 0..numPartitions-1
- * @param prefix the prefix that we have already checked against
- * @param maxDepth the maximum depth we will build a trie for
- * @return the trie node that will divide the splits correctly
- */
- private static TrieNode buildTrie(Text[] splits, int lower, int upper,
- Text prefix, int maxDepth) {
- int depth = prefix.getLength();
- if (depth >= maxDepth || lower == upper) {
- return new LeafTrieNode(depth, splits, lower, upper);
- }
- InnerTrieNode result = new InnerTrieNode(depth);
- Text trial = new Text(prefix);
- // append an extra byte on to the prefix
- trial.append(new byte[1], 0, 1);
- int currentBound = lower;
- for(int ch = 0; ch < 255; ++ch) {
- trial.getBytes()[depth] = (byte) (ch + 1);
- lower = currentBound;
- while (currentBound < upper) {
- if (splits[currentBound].compareTo(trial) >= 0) {
- break;
- }
- currentBound += 1;
- }
- trial.getBytes()[depth] = (byte) ch;
- result.child[ch] = buildTrie(splits, lower, currentBound, trial,
- maxDepth);
- }
- // pick up the rest
- trial.getBytes()[depth] = (byte) 255;
- result.child[255] = buildTrie(splits, currentBound, upper, trial,
- maxDepth);
- return result;
- }
-
- public void setConf(Configuration conf) {
- try {
- FileSystem fs = FileSystem.getLocal(conf);
- this.conf = conf;
- Path partFile = new Path(TeraInputFormat.PARTITION_FILENAME);
- splitPoints = readPartitions(fs, partFile, conf);
- trie = buildTrie(splitPoints, 0, splitPoints.length, new Text(), 2);
- } catch (IOException ie) {
- throw new IllegalArgumentException("can't read paritions file", ie);
- }
- }
-
- public Configuration getConf() {
- return conf;
- }
-
- public TotalOrderPartitioner() {
- }
-
- public int getPartition(Text key, Text value, int numPartitions) {
- return trie.findPartition(key);
- }
-
- }
-
- /**
- * A total order partitioner that assigns keys based on their first
- * PREFIX_LENGTH bytes, assuming a flat distribution.
- */
- public static class SimplePartitioner extends Partitioner<Text, Text>
- implements Configurable {
- int prefixesPerReduce;
- private static final int PREFIX_LENGTH = 3;
- private Configuration conf = null;
- public void setConf(Configuration conf) {
- this.conf = conf;
- prefixesPerReduce = (int) Math.ceil((1 << (8 * PREFIX_LENGTH)) /
- (float) conf.getInt(MRJobConfig.NUM_REDUCES, 1));
- }
-
- public Configuration getConf() {
- return conf;
- }
-
- @Override
- public int getPartition(Text key, Text value, int numPartitions) {
- byte[] bytes = key.getBytes();
- int len = Math.min(PREFIX_LENGTH, key.getLength());
- int prefix = 0;
- for(int i=0; i < len; ++i) {
- prefix = (prefix << 8) | (0xff & bytes[i]);
- }
- return prefix / prefixesPerReduce;
- }
- }
-
- public static boolean getUseSimplePartitioner(JobContext job) {
- return job.getConfiguration().getBoolean(SIMPLE_PARTITIONER, false);
- }
-
- public static void setUseSimplePartitioner(Job job, boolean value) {
- job.getConfiguration().setBoolean(SIMPLE_PARTITIONER, value);
- }
-
- public static int getOutputReplication(JobContext job) {
- return job.getConfiguration().getInt(OUTPUT_REPLICATION, 1);
- }
-
- public static void setOutputReplication(Job job, int value) {
- job.getConfiguration().setInt(OUTPUT_REPLICATION, value);
- }
-
- public int run(String[] args) throws Exception {
- if (args.length != 2) {
- System.err.println("Invalid no. of arguments provided");
- System.err.println("Usage: terasort <input-dir> <output-dir>");
- return -1;
- }
-
- LOG.info("starting");
- Job job = Job.getInstance(getConf());
- Path inputDir = new Path(args[0]);
- Path outputDir = new Path(args[1]);
- boolean useSimplePartitioner = getUseSimplePartitioner(job);
- TeraInputFormat.setInputPaths(job, inputDir);
- FileOutputFormat.setOutputPath(job, outputDir);
- job.setJobName("TeraSort");
- job.setJarByClass(TeraSort.class);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(Text.class);
- job.setInputFormatClass(TeraInputFormat.class);
- job.setOutputFormatClass(TeraOutputFormat.class);
- if (useSimplePartitioner) {
- job.setPartitionerClass(SimplePartitioner.class);
- } else {
- long start = System.currentTimeMillis();
- Path partitionFile = new Path(outputDir,
- TeraInputFormat.PARTITION_FILENAME);
- URI partitionUri = new URI(partitionFile.toString() +
- "#" + TeraInputFormat.PARTITION_FILENAME);
- try {
- TeraInputFormat.writePartitionFile(job, partitionFile);
- } catch (Throwable e) {
- LOG.error(e.getMessage());
- return -1;
- }
- job.addCacheFile(partitionUri);
- long end = System.currentTimeMillis();
- System.out.println("Spent " + (end - start) + "ms computing partitions.");
- job.setPartitionerClass(TotalOrderPartitioner.class);
- }
-
- job.getConfiguration().setInt("dfs.replication", getOutputReplication(job));
- TeraOutputFormat.setFinalSync(job, true);
- int ret = job.waitForCompletion(true) ? 0 : 1;
- LOG.info("done");
- return ret;
- }
-
- /**
- * @param args
- */
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new TeraSort(), args);
- System.exit(res);
- }
-
-}
[03/10] TEZ-1055. Rename tez-mapreduce-examples to tez-examples
(Hitesh Shah via bikas)
Posted by bi...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MapredWordCount.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MapredWordCount.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MapredWordCount.java
new file mode 100644
index 0000000..eef6223
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MapredWordCount.java
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.StringTokenizer;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * This is an example Hadoop Map/Reduce application using the mapred apis.
+ * It reads the text input files, breaks each line into words
+ * and counts them. The output is a locally sorted list of words and the
+ * count of how often they occurred.
+ *
+ * To run: bin/hadoop jar examples.jar wordcount
+ * [-m <i>maps</i>] [-r <i>reduces</i>] <i>in-dir</i> <i>out-dir</i>
+ */
+public class MapredWordCount extends Configured implements Tool {
+
+ private static final Log LOG = LogFactory.getLog(MapredWordCount.class);
+
+ /**
+ * Counts the words in each line.
+ * For each line of input, break the line into words and emit them as
+ * (<b>word</b>, <b>1</b>).
+ */
+ public static class MapClass extends MapReduceBase
+ implements Mapper<LongWritable, Text, Text, IntWritable> {
+
+ private final static IntWritable one = new IntWritable(1);
+ private Text word = new Text();
+
+ public void map(LongWritable key, Text value,
+ OutputCollector<Text, IntWritable> output,
+ Reporter reporter) throws IOException {
+ String line = value.toString();
+ StringTokenizer itr = new StringTokenizer(line);
+ while (itr.hasMoreTokens()) {
+ word.set(itr.nextToken());
+ output.collect(word, one);
+ }
+ }
+ }
+
+ /**
+ * A reducer class that just emits the sum of the input values.
+ */
+ public static class Reduce extends MapReduceBase
+ implements Reducer<Text, IntWritable, Text, IntWritable> {
+
+ public void reduce(Text key, Iterator<IntWritable> values,
+ OutputCollector<Text, IntWritable> output,
+ Reporter reporter) throws IOException {
+ int sum = 0;
+ while (values.hasNext()) {
+ sum += values.next().get();
+ }
+ output.collect(key, new IntWritable(sum));
+ }
+ }
+
+ static int printUsage() {
+ System.out.println("wordcount [-m <maps>] [-r <reduces>] <input> <output>");
+ ToolRunner.printGenericCommandUsage(System.out);
+ return -1;
+ }
+
+ /**
+ * The main driver for word count map/reduce program.
+ * Invoke this method to submit the map/reduce job.
+ * @throws IOException When there is communication problems with the
+ * job tracker.
+ */
+ public int run(String[] args) throws Exception {
+ JobConf conf = new JobConf(getConf(), MapredWordCount.class);
+ conf.setJobName("wordcount");
+ LOG.info("Running WordCount job using mapred apis");
+
+ // the keys are words (strings)
+ conf.setOutputKeyClass(Text.class);
+ // the values are counts (ints)
+ conf.setOutputValueClass(IntWritable.class);
+
+ conf.setMapperClass(MapClass.class);
+ conf.setCombinerClass(Reduce.class);
+ conf.setReducerClass(Reduce.class);
+
+ List<String> other_args = new ArrayList<String>();
+ for(int i=0; i < args.length; ++i) {
+ try {
+ if ("-m".equals(args[i])) {
+ conf.setNumMapTasks(Integer.parseInt(args[++i]));
+ } else if ("-r".equals(args[i])) {
+ conf.setNumReduceTasks(Integer.parseInt(args[++i]));
+ } else {
+ other_args.add(args[i]);
+ }
+ } catch (NumberFormatException except) {
+ LOG.error("Integer expected instead of " + args[i]);
+ return printUsage();
+ } catch (ArrayIndexOutOfBoundsException except) {
+ LOG.error("Required parameter missing from " + args[i-1]);
+ return printUsage();
+ }
+ }
+ // Make sure there are exactly 2 parameters left.
+ if (other_args.size() != 2) {
+ LOG.error("Wrong number of parameters: " +
+ other_args.size() + " instead of 2.");
+ return printUsage();
+ }
+ FileInputFormat.setInputPaths(conf, other_args.get(0));
+ FileOutputFormat.setOutputPath(conf, new Path(other_args.get(1)));
+
+ JobClient.runJob(conf);
+ return 0;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(),
+ new MapredWordCount(), args);
+ System.exit(res);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java
new file mode 100644
index 0000000..8251b78
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java
@@ -0,0 +1,757 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.ClusterStatus;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * This program uses map/reduce to just run a distributed job where there is
+ * no interaction between the tasks and each task writes a large unsorted
+ * random sequence of words.
+ * In order for this program to generate data for terasort with a 5-10 words
+ * per key and 20-100 words per value, have the following config:
+ * <xmp>
+ * <?xml version="1.0"?>
+ * <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+ * <configuration>
+ * <property>
+ * <name>mapreduce.randomtextwriter.minwordskey</name>
+ * <value>5</value>
+ * </property>
+ * <property>
+ * <name>mapreduce.randomtextwriter.maxwordskey</name>
+ * <value>10</value>
+ * </property>
+ * <property>
+ * <name>mapreduce.randomtextwriter.minwordsvalue</name>
+ * <value>20</value>
+ * </property>
+ * <property>
+ * <name>mapreduce.randomtextwriter.maxwordsvalue</name>
+ * <value>100</value>
+ * </property>
+ * <property>
+ * <name>mapreduce.randomtextwriter.totalbytes</name>
+ * <value>1099511627776</value>
+ * </property>
+ * </configuration></xmp>
+ *
+ * Equivalently, {@link RandomTextWriter} also supports all the above options
+ * and ones supported by {@link Tool} via the command-line.
+ *
+ * To run: bin/hadoop jar hadoop-${version}-examples.jar randomtextwriter
+ * [-outFormat <i>output format class</i>] <i>output</i>
+ */
+public class RandomTextWriter extends Configured implements Tool {
+ public static final String TOTAL_BYTES =
+ "mapreduce.randomtextwriter.totalbytes";
+ public static final String BYTES_PER_MAP =
+ "mapreduce.randomtextwriter.bytespermap";
+ public static final String MAPS_PER_HOST =
+ "mapreduce.randomtextwriter.mapsperhost";
+ public static final String MAX_VALUE = "mapreduce.randomtextwriter.maxwordsvalue";
+ public static final String MIN_VALUE = "mapreduce.randomtextwriter.minwordsvalue";
+ public static final String MIN_KEY = "mapreduce.randomtextwriter.minwordskey";
+ public static final String MAX_KEY = "mapreduce.randomtextwriter.maxwordskey";
+
+ static int printUsage() {
+ System.out.println("randomtextwriter " +
+ "[-outFormat <output format class>] " +
+ "<output>");
+ ToolRunner.printGenericCommandUsage(System.out);
+ return 2;
+ }
+
+ /**
+ * User counters
+ */
+ static enum Counters { RECORDS_WRITTEN, BYTES_WRITTEN }
+
+ static class RandomTextMapper extends Mapper<Text, Text, Text, Text> {
+
+ private long numBytesToWrite;
+ private int minWordsInKey;
+ private int wordsInKeyRange;
+ private int minWordsInValue;
+ private int wordsInValueRange;
+ private Random random = new Random();
+
+ /**
+ * Save the configuration value that we need to write the data.
+ */
+ public void setup(Context context) {
+ Configuration conf = context.getConfiguration();
+ numBytesToWrite = conf.getLong(BYTES_PER_MAP,
+ 1*1024*1024*1024);
+ minWordsInKey = conf.getInt(MIN_KEY, 5);
+ wordsInKeyRange = (conf.getInt(MAX_KEY, 10) - minWordsInKey);
+ minWordsInValue = conf.getInt(MIN_VALUE, 10);
+ wordsInValueRange = (conf.getInt(MAX_VALUE, 100) - minWordsInValue);
+ }
+
+ /**
+ * Given an output filename, write a bunch of random records to it.
+ */
+ public void map(Text key, Text value,
+ Context context) throws IOException,InterruptedException {
+ int itemCount = 0;
+ while (numBytesToWrite > 0) {
+ // Generate the key/value
+ int noWordsKey = minWordsInKey +
+ (wordsInKeyRange != 0 ? random.nextInt(wordsInKeyRange) : 0);
+ int noWordsValue = minWordsInValue +
+ (wordsInValueRange != 0 ? random.nextInt(wordsInValueRange) : 0);
+ Text keyWords = generateSentence(noWordsKey);
+ Text valueWords = generateSentence(noWordsValue);
+
+ // Write the sentence
+ context.write(keyWords, valueWords);
+
+ numBytesToWrite -= (keyWords.getLength() + valueWords.getLength());
+
+ // Update counters, progress etc.
+ context.getCounter(Counters.BYTES_WRITTEN).increment(
+ keyWords.getLength() + valueWords.getLength());
+ context.getCounter(Counters.RECORDS_WRITTEN).increment(1);
+ if (++itemCount % 200 == 0) {
+ context.setStatus("wrote record " + itemCount + ". " +
+ numBytesToWrite + " bytes left.");
+ }
+ }
+ context.setStatus("done with " + itemCount + " records.");
+ }
+
+ private Text generateSentence(int noWords) {
+ StringBuffer sentence = new StringBuffer();
+ String space = " ";
+ for (int i=0; i < noWords; ++i) {
+ sentence.append(words[random.nextInt(words.length)]);
+ sentence.append(space);
+ }
+ return new Text(sentence.toString());
+ }
+ }
+
+ /**
+ * This is the main routine for launching a distributed random write job.
+ * It runs 10 maps/node and each node writes 1 gig of data to a DFS file.
+ * The reduce doesn't do anything.
+ *
+ * @throws IOException
+ */
+ public int run(String[] args) throws Exception {
+ if (args.length == 0) {
+ return printUsage();
+ }
+
+ Configuration conf = getConf();
+ JobClient client = new JobClient(conf);
+ ClusterStatus cluster = client.getClusterStatus();
+ int numMapsPerHost = conf.getInt(MAPS_PER_HOST, 10);
+ long numBytesToWritePerMap = conf.getLong(BYTES_PER_MAP,
+ 1*1024*1024*1024);
+ if (numBytesToWritePerMap == 0) {
+ System.err.println("Cannot have " + BYTES_PER_MAP +" set to 0");
+ return -2;
+ }
+ long totalBytesToWrite = conf.getLong(TOTAL_BYTES,
+ numMapsPerHost*numBytesToWritePerMap*cluster.getTaskTrackers());
+ int numMaps = (int) (totalBytesToWrite / numBytesToWritePerMap);
+ if (numMaps == 0 && totalBytesToWrite > 0) {
+ numMaps = 1;
+ conf.setLong(BYTES_PER_MAP, totalBytesToWrite);
+ }
+ conf.setInt(MRJobConfig.NUM_MAPS, numMaps);
+
+ Job job = new Job(conf);
+
+ job.setJarByClass(RandomTextWriter.class);
+ job.setJobName("random-text-writer");
+
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(Text.class);
+
+ job.setInputFormatClass(RandomWriter.RandomInputFormat.class);
+ job.setMapperClass(RandomTextMapper.class);
+
+ Class<? extends OutputFormat> outputFormatClass =
+ SequenceFileOutputFormat.class;
+ List<String> otherArgs = new ArrayList<String>();
+ for(int i=0; i < args.length; ++i) {
+ try {
+ if ("-outFormat".equals(args[i])) {
+ outputFormatClass =
+ Class.forName(args[++i]).asSubclass(OutputFormat.class);
+ } else {
+ otherArgs.add(args[i]);
+ }
+ } catch (ArrayIndexOutOfBoundsException except) {
+ System.out.println("ERROR: Required parameter missing from " +
+ args[i-1]);
+ return printUsage(); // exits
+ }
+ }
+
+ job.setOutputFormatClass(outputFormatClass);
+ FileOutputFormat.setOutputPath(job, new Path(otherArgs.get(0)));
+
+ System.out.println("Running " + numMaps + " maps.");
+
+ // reducer NONE
+ job.setNumReduceTasks(0);
+
+ Date startTime = new Date();
+ System.out.println("Job started: " + startTime);
+ int ret = job.waitForCompletion(true) ? 0 : 1;
+ Date endTime = new Date();
+ System.out.println("Job ended: " + endTime);
+ System.out.println("The job took " +
+ (endTime.getTime() - startTime.getTime()) /1000 +
+ " seconds.");
+
+ return ret;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(), new RandomTextWriter(), args);
+ System.exit(res);
+ }
+
+ /**
+ * A random list of 100 words from /usr/share/dict/words
+ */
+ private static String[] words = {
+ "diurnalness", "Homoiousian",
+ "spiranthic", "tetragynian",
+ "silverhead", "ungreat",
+ "lithograph", "exploiter",
+ "physiologian", "by",
+ "hellbender", "Filipendula",
+ "undeterring", "antiscolic",
+ "pentagamist", "hypoid",
+ "cacuminal", "sertularian",
+ "schoolmasterism", "nonuple",
+ "gallybeggar", "phytonic",
+ "swearingly", "nebular",
+ "Confervales", "thermochemically",
+ "characinoid", "cocksuredom",
+ "fallacious", "feasibleness",
+ "debromination", "playfellowship",
+ "tramplike", "testa",
+ "participatingly", "unaccessible",
+ "bromate", "experientialist",
+ "roughcast", "docimastical",
+ "choralcelo", "blightbird",
+ "peptonate", "sombreroed",
+ "unschematized", "antiabolitionist",
+ "besagne", "mastication",
+ "bromic", "sviatonosite",
+ "cattimandoo", "metaphrastical",
+ "endotheliomyoma", "hysterolysis",
+ "unfulminated", "Hester",
+ "oblongly", "blurredness",
+ "authorling", "chasmy",
+ "Scorpaenidae", "toxihaemia",
+ "Dictograph", "Quakerishly",
+ "deaf", "timbermonger",
+ "strammel", "Thraupidae",
+ "seditious", "plerome",
+ "Arneb", "eristically",
+ "serpentinic", "glaumrie",
+ "socioromantic", "apocalypst",
+ "tartrous", "Bassaris",
+ "angiolymphoma", "horsefly",
+ "kenno", "astronomize",
+ "euphemious", "arsenide",
+ "untongued", "parabolicness",
+ "uvanite", "helpless",
+ "gemmeous", "stormy",
+ "templar", "erythrodextrin",
+ "comism", "interfraternal",
+ "preparative", "parastas",
+ "frontoorbital", "Ophiosaurus",
+ "diopside", "serosanguineous",
+ "ununiformly", "karyological",
+ "collegian", "allotropic",
+ "depravity", "amylogenesis",
+ "reformatory", "epidymides",
+ "pleurotropous", "trillium",
+ "dastardliness", "coadvice",
+ "embryotic", "benthonic",
+ "pomiferous", "figureheadship",
+ "Megaluridae", "Harpa",
+ "frenal", "commotion",
+ "abthainry", "cobeliever",
+ "manilla", "spiciferous",
+ "nativeness", "obispo",
+ "monilioid", "biopsic",
+ "valvula", "enterostomy",
+ "planosubulate", "pterostigma",
+ "lifter", "triradiated",
+ "venialness", "tum",
+ "archistome", "tautness",
+ "unswanlike", "antivenin",
+ "Lentibulariaceae", "Triphora",
+ "angiopathy", "anta",
+ "Dawsonia", "becomma",
+ "Yannigan", "winterproof",
+ "antalgol", "harr",
+ "underogating", "ineunt",
+ "cornberry", "flippantness",
+ "scyphostoma", "approbation",
+ "Ghent", "Macraucheniidae",
+ "scabbiness", "unanatomized",
+ "photoelasticity", "eurythermal",
+ "enation", "prepavement",
+ "flushgate", "subsequentially",
+ "Edo", "antihero",
+ "Isokontae", "unforkedness",
+ "porriginous", "daytime",
+ "nonexecutive", "trisilicic",
+ "morphiomania", "paranephros",
+ "botchedly", "impugnation",
+ "Dodecatheon", "obolus",
+ "unburnt", "provedore",
+ "Aktistetae", "superindifference",
+ "Alethea", "Joachimite",
+ "cyanophilous", "chorograph",
+ "brooky", "figured",
+ "periclitation", "quintette",
+ "hondo", "ornithodelphous",
+ "unefficient", "pondside",
+ "bogydom", "laurinoxylon",
+ "Shiah", "unharmed",
+ "cartful", "noncrystallized",
+ "abusiveness", "cromlech",
+ "japanned", "rizzomed",
+ "underskin", "adscendent",
+ "allectory", "gelatinousness",
+ "volcano", "uncompromisingly",
+ "cubit", "idiotize",
+ "unfurbelowed", "undinted",
+ "magnetooptics", "Savitar",
+ "diwata", "ramosopalmate",
+ "Pishquow", "tomorn",
+ "apopenptic", "Haversian",
+ "Hysterocarpus", "ten",
+ "outhue", "Bertat",
+ "mechanist", "asparaginic",
+ "velaric", "tonsure",
+ "bubble", "Pyrales",
+ "regardful", "glyphography",
+ "calabazilla", "shellworker",
+ "stradametrical", "havoc",
+ "theologicopolitical", "sawdust",
+ "diatomaceous", "jajman",
+ "temporomastoid", "Serrifera",
+ "Ochnaceae", "aspersor",
+ "trailmaking", "Bishareen",
+ "digitule", "octogynous",
+ "epididymitis", "smokefarthings",
+ "bacillite", "overcrown",
+ "mangonism", "sirrah",
+ "undecorated", "psychofugal",
+ "bismuthiferous", "rechar",
+ "Lemuridae", "frameable",
+ "thiodiazole", "Scanic",
+ "sportswomanship", "interruptedness",
+ "admissory", "osteopaedion",
+ "tingly", "tomorrowness",
+ "ethnocracy", "trabecular",
+ "vitally", "fossilism",
+ "adz", "metopon",
+ "prefatorial", "expiscate",
+ "diathermacy", "chronist",
+ "nigh", "generalizable",
+ "hysterogen", "aurothiosulphuric",
+ "whitlowwort", "downthrust",
+ "Protestantize", "monander",
+ "Itea", "chronographic",
+ "silicize", "Dunlop",
+ "eer", "componental",
+ "spot", "pamphlet",
+ "antineuritic", "paradisean",
+ "interruptor", "debellator",
+ "overcultured", "Florissant",
+ "hyocholic", "pneumatotherapy",
+ "tailoress", "rave",
+ "unpeople", "Sebastian",
+ "thermanesthesia", "Coniferae",
+ "swacking", "posterishness",
+ "ethmopalatal", "whittle",
+ "analgize", "scabbardless",
+ "naught", "symbiogenetically",
+ "trip", "parodist",
+ "columniform", "trunnel",
+ "yawler", "goodwill",
+ "pseudohalogen", "swangy",
+ "cervisial", "mediateness",
+ "genii", "imprescribable",
+ "pony", "consumptional",
+ "carposporangial", "poleax",
+ "bestill", "subfebrile",
+ "sapphiric", "arrowworm",
+ "qualminess", "ultraobscure",
+ "thorite", "Fouquieria",
+ "Bermudian", "prescriber",
+ "elemicin", "warlike",
+ "semiangle", "rotular",
+ "misthread", "returnability",
+ "seraphism", "precostal",
+ "quarried", "Babylonism",
+ "sangaree", "seelful",
+ "placatory", "pachydermous",
+ "bozal", "galbulus",
+ "spermaphyte", "cumbrousness",
+ "pope", "signifier",
+ "Endomycetaceae", "shallowish",
+ "sequacity", "periarthritis",
+ "bathysphere", "pentosuria",
+ "Dadaism", "spookdom",
+ "Consolamentum", "afterpressure",
+ "mutter", "louse",
+ "ovoviviparous", "corbel",
+ "metastoma", "biventer",
+ "Hydrangea", "hogmace",
+ "seizing", "nonsuppressed",
+ "oratorize", "uncarefully",
+ "benzothiofuran", "penult",
+ "balanocele", "macropterous",
+ "dishpan", "marten",
+ "absvolt", "jirble",
+ "parmelioid", "airfreighter",
+ "acocotl", "archesporial",
+ "hypoplastral", "preoral",
+ "quailberry", "cinque",
+ "terrestrially", "stroking",
+ "limpet", "moodishness",
+ "canicule", "archididascalian",
+ "pompiloid", "overstaid",
+ "introducer", "Italical",
+ "Christianopaganism", "prescriptible",
+ "subofficer", "danseuse",
+ "cloy", "saguran",
+ "frictionlessly", "deindividualization",
+ "Bulanda", "ventricous",
+ "subfoliar", "basto",
+ "scapuloradial", "suspend",
+ "stiffish", "Sphenodontidae",
+ "eternal", "verbid",
+ "mammonish", "upcushion",
+ "barkometer", "concretion",
+ "preagitate", "incomprehensible",
+ "tristich", "visceral",
+ "hemimelus", "patroller",
+ "stentorophonic", "pinulus",
+ "kerykeion", "brutism",
+ "monstership", "merciful",
+ "overinstruct", "defensibly",
+ "bettermost", "splenauxe",
+ "Mormyrus", "unreprimanded",
+ "taver", "ell",
+ "proacquittal", "infestation",
+ "overwoven", "Lincolnlike",
+ "chacona", "Tamil",
+ "classificational", "lebensraum",
+ "reeveland", "intuition",
+ "Whilkut", "focaloid",
+ "Eleusinian", "micromembrane",
+ "byroad", "nonrepetition",
+ "bacterioblast", "brag",
+ "ribaldrous", "phytoma",
+ "counteralliance", "pelvimetry",
+ "pelf", "relaster",
+ "thermoresistant", "aneurism",
+ "molossic", "euphonym",
+ "upswell", "ladhood",
+ "phallaceous", "inertly",
+ "gunshop", "stereotypography",
+ "laryngic", "refasten",
+ "twinling", "oflete",
+ "hepatorrhaphy", "electrotechnics",
+ "cockal", "guitarist",
+ "topsail", "Cimmerianism",
+ "larklike", "Llandovery",
+ "pyrocatechol", "immatchable",
+ "chooser", "metrocratic",
+ "craglike", "quadrennial",
+ "nonpoisonous", "undercolored",
+ "knob", "ultratense",
+ "balladmonger", "slait",
+ "sialadenitis", "bucketer",
+ "magnificently", "unstipulated",
+ "unscourged", "unsupercilious",
+ "packsack", "pansophism",
+ "soorkee", "percent",
+ "subirrigate", "champer",
+ "metapolitics", "spherulitic",
+ "involatile", "metaphonical",
+ "stachyuraceous", "speckedness",
+ "bespin", "proboscidiform",
+ "gul", "squit",
+ "yeelaman", "peristeropode",
+ "opacousness", "shibuichi",
+ "retinize", "yote",
+ "misexposition", "devilwise",
+ "pumpkinification", "vinny",
+ "bonze", "glossing",
+ "decardinalize", "transcortical",
+ "serphoid", "deepmost",
+ "guanajuatite", "wemless",
+ "arval", "lammy",
+ "Effie", "Saponaria",
+ "tetrahedral", "prolificy",
+ "excerpt", "dunkadoo",
+ "Spencerism", "insatiately",
+ "Gilaki", "oratorship",
+ "arduousness", "unbashfulness",
+ "Pithecolobium", "unisexuality",
+ "veterinarian", "detractive",
+ "liquidity", "acidophile",
+ "proauction", "sural",
+ "totaquina", "Vichyite",
+ "uninhabitedness", "allegedly",
+ "Gothish", "manny",
+ "Inger", "flutist",
+ "ticktick", "Ludgatian",
+ "homotransplant", "orthopedical",
+ "diminutively", "monogoneutic",
+ "Kenipsim", "sarcologist",
+ "drome", "stronghearted",
+ "Fameuse", "Swaziland",
+ "alen", "chilblain",
+ "beatable", "agglomeratic",
+ "constitutor", "tendomucoid",
+ "porencephalous", "arteriasis",
+ "boser", "tantivy",
+ "rede", "lineamental",
+ "uncontradictableness", "homeotypical",
+ "masa", "folious",
+ "dosseret", "neurodegenerative",
+ "subtransverse", "Chiasmodontidae",
+ "palaeotheriodont", "unstressedly",
+ "chalcites", "piquantness",
+ "lampyrine", "Aplacentalia",
+ "projecting", "elastivity",
+ "isopelletierin", "bladderwort",
+ "strander", "almud",
+ "iniquitously", "theologal",
+ "bugre", "chargeably",
+ "imperceptivity", "meriquinoidal",
+ "mesophyte", "divinator",
+ "perfunctory", "counterappellant",
+ "synovial", "charioteer",
+ "crystallographical", "comprovincial",
+ "infrastapedial", "pleasurehood",
+ "inventurous", "ultrasystematic",
+ "subangulated", "supraoesophageal",
+ "Vaishnavism", "transude",
+ "chrysochrous", "ungrave",
+ "reconciliable", "uninterpleaded",
+ "erlking", "wherefrom",
+ "aprosopia", "antiadiaphorist",
+ "metoxazine", "incalculable",
+ "umbellic", "predebit",
+ "foursquare", "unimmortal",
+ "nonmanufacture", "slangy",
+ "predisputant", "familist",
+ "preaffiliate", "friarhood",
+ "corelysis", "zoonitic",
+ "halloo", "paunchy",
+ "neuromimesis", "aconitine",
+ "hackneyed", "unfeeble",
+ "cubby", "autoschediastical",
+ "naprapath", "lyrebird",
+ "inexistency", "leucophoenicite",
+ "ferrogoslarite", "reperuse",
+ "uncombable", "tambo",
+ "propodiale", "diplomatize",
+ "Russifier", "clanned",
+ "corona", "michigan",
+ "nonutilitarian", "transcorporeal",
+ "bought", "Cercosporella",
+ "stapedius", "glandularly",
+ "pictorially", "weism",
+ "disilane", "rainproof",
+ "Caphtor", "scrubbed",
+ "oinomancy", "pseudoxanthine",
+ "nonlustrous", "redesertion",
+ "Oryzorictinae", "gala",
+ "Mycogone", "reappreciate",
+ "cyanoguanidine", "seeingness",
+ "breadwinner", "noreast",
+ "furacious", "epauliere",
+ "omniscribent", "Passiflorales",
+ "uninductive", "inductivity",
+ "Orbitolina", "Semecarpus",
+ "migrainoid", "steprelationship",
+ "phlogisticate", "mesymnion",
+ "sloped", "edificator",
+ "beneficent", "culm",
+ "paleornithology", "unurban",
+ "throbless", "amplexifoliate",
+ "sesquiquintile", "sapience",
+ "astucious", "dithery",
+ "boor", "ambitus",
+ "scotching", "uloid",
+ "uncompromisingness", "hoove",
+ "waird", "marshiness",
+ "Jerusalem", "mericarp",
+ "unevoked", "benzoperoxide",
+ "outguess", "pyxie",
+ "hymnic", "euphemize",
+ "mendacity", "erythremia",
+ "rosaniline", "unchatteled",
+ "lienteria", "Bushongo",
+ "dialoguer", "unrepealably",
+ "rivethead", "antideflation",
+ "vinegarish", "manganosiderite",
+ "doubtingness", "ovopyriform",
+ "Cephalodiscus", "Muscicapa",
+ "Animalivora", "angina",
+ "planispheric", "ipomoein",
+ "cuproiodargyrite", "sandbox",
+ "scrat", "Munnopsidae",
+ "shola", "pentafid",
+ "overstudiousness", "times",
+ "nonprofession", "appetible",
+ "valvulotomy", "goladar",
+ "uniarticular", "oxyterpene",
+ "unlapsing", "omega",
+ "trophonema", "seminonflammable",
+ "circumzenithal", "starer",
+ "depthwise", "liberatress",
+ "unleavened", "unrevolting",
+ "groundneedle", "topline",
+ "wandoo", "umangite",
+ "ordinant", "unachievable",
+ "oversand", "snare",
+ "avengeful", "unexplicit",
+ "mustafina", "sonable",
+ "rehabilitative", "eulogization",
+ "papery", "technopsychology",
+ "impressor", "cresylite",
+ "entame", "transudatory",
+ "scotale", "pachydermatoid",
+ "imaginary", "yeat",
+ "slipped", "stewardship",
+ "adatom", "cockstone",
+ "skyshine", "heavenful",
+ "comparability", "exprobratory",
+ "dermorhynchous", "parquet",
+ "cretaceous", "vesperal",
+ "raphis", "undangered",
+ "Glecoma", "engrain",
+ "counteractively", "Zuludom",
+ "orchiocatabasis", "Auriculariales",
+ "warriorwise", "extraorganismal",
+ "overbuilt", "alveolite",
+ "tetchy", "terrificness",
+ "widdle", "unpremonished",
+ "rebilling", "sequestrum",
+ "equiconvex", "heliocentricism",
+ "catabaptist", "okonite",
+ "propheticism", "helminthagogic",
+ "calycular", "giantly",
+ "wingable", "golem",
+ "unprovided", "commandingness",
+ "greave", "haply",
+ "doina", "depressingly",
+ "subdentate", "impairment",
+ "decidable", "neurotrophic",
+ "unpredict", "bicorporeal",
+ "pendulant", "flatman",
+ "intrabred", "toplike",
+ "Prosobranchiata", "farrantly",
+ "toxoplasmosis", "gorilloid",
+ "dipsomaniacal", "aquiline",
+ "atlantite", "ascitic",
+ "perculsive", "prospectiveness",
+ "saponaceous", "centrifugalization",
+ "dinical", "infravaginal",
+ "beadroll", "affaite",
+ "Helvidian", "tickleproof",
+ "abstractionism", "enhedge",
+ "outwealth", "overcontribute",
+ "coldfinch", "gymnastic",
+ "Pincian", "Munychian",
+ "codisjunct", "quad",
+ "coracomandibular", "phoenicochroite",
+ "amender", "selectivity",
+ "putative", "semantician",
+ "lophotrichic", "Spatangoidea",
+ "saccharogenic", "inferent",
+ "Triconodonta", "arrendation",
+ "sheepskin", "taurocolla",
+ "bunghole", "Machiavel",
+ "triakistetrahedral", "dehairer",
+ "prezygapophysial", "cylindric",
+ "pneumonalgia", "sleigher",
+ "emir", "Socraticism",
+ "licitness", "massedly",
+ "instructiveness", "sturdied",
+ "redecrease", "starosta",
+ "evictor", "orgiastic",
+ "squdge", "meloplasty",
+ "Tsonecan", "repealableness",
+ "swoony", "myesthesia",
+ "molecule", "autobiographist",
+ "reciprocation", "refective",
+ "unobservantness", "tricae",
+ "ungouged", "floatability",
+ "Mesua", "fetlocked",
+ "chordacentrum", "sedentariness",
+ "various", "laubanite",
+ "nectopod", "zenick",
+ "sequentially", "analgic",
+ "biodynamics", "posttraumatic",
+ "nummi", "pyroacetic",
+ "bot", "redescend",
+ "dispermy", "undiffusive",
+ "circular", "trillion",
+ "Uraniidae", "ploration",
+ "discipular", "potentness",
+ "sud", "Hu",
+ "Eryon", "plugger",
+ "subdrainage", "jharal",
+ "abscission", "supermarket",
+ "countergabion", "glacierist",
+ "lithotresis", "minniebush",
+ "zanyism", "eucalypteol",
+ "sterilely", "unrealize",
+ "unpatched", "hypochondriacism",
+ "critically", "cheesecutter",
+ };
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomWriter.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomWriter.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomWriter.java
new file mode 100644
index 0000000..c9b51c7
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomWriter.java
@@ -0,0 +1,298 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapred.ClusterStatus;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * This program uses map/reduce to just run a distributed job where there is
+ * no interaction between the tasks and each task write a large unsorted
+ * random binary sequence file of BytesWritable.
+ * In order for this program to generate data for terasort with 10-byte keys
+ * and 90-byte values, have the following config:
+ * <xmp>
+ * <?xml version="1.0"?>
+ * <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+ * <configuration>
+ * <property>
+ * <name>mapreduce.randomwriter.minkey</name>
+ * <value>10</value>
+ * </property>
+ * <property>
+ * <name>mapreduce.randomwriter.maxkey</name>
+ * <value>10</value>
+ * </property>
+ * <property>
+ * <name>mapreduce.randomwriter.minvalue</name>
+ * <value>90</value>
+ * </property>
+ * <property>
+ * <name>mapreduce.randomwriter.maxvalue</name>
+ * <value>90</value>
+ * </property>
+ * <property>
+ * <name>mapreduce.randomwriter.totalbytes</name>
+ * <value>1099511627776</value>
+ * </property>
+ * </configuration></xmp>
+ *
+ * Equivalently, {@link RandomWriter} also supports all the above options
+ * and ones supported by {@link GenericOptionsParser} via the command-line.
+ */
+public class RandomWriter extends Configured implements Tool {
+ public static final String TOTAL_BYTES = "mapreduce.randomwriter.totalbytes";
+ public static final String BYTES_PER_MAP =
+ "mapreduce.randomwriter.bytespermap";
+ public static final String MAPS_PER_HOST =
+ "mapreduce.randomwriter.mapsperhost";
+ public static final String MAX_VALUE = "mapreduce.randomwriter.maxvalue";
+ public static final String MIN_VALUE = "mapreduce.randomwriter.minvalue";
+ public static final String MIN_KEY = "mapreduce.randomwriter.minkey";
+ public static final String MAX_KEY = "mapreduce.randomwriter.maxkey";
+
+ /**
+ * User counters
+ */
+ static enum Counters { RECORDS_WRITTEN, BYTES_WRITTEN }
+
+ /**
+ * A custom input format that creates virtual inputs of a single string
+ * for each map.
+ */
+ static class RandomInputFormat extends InputFormat<Text, Text> {
+
+ /**
+ * Generate the requested number of file splits, with the filename
+ * set to the filename of the output file.
+ */
+ public List<InputSplit> getSplits(JobContext job) throws IOException {
+ List<InputSplit> result = new ArrayList<InputSplit>();
+ Path outDir = FileOutputFormat.getOutputPath(job);
+ int numSplits =
+ job.getConfiguration().getInt(MRJobConfig.NUM_MAPS, 1);
+ for(int i=0; i < numSplits; ++i) {
+ result.add(new FileSplit(new Path(outDir, "dummy-split-" + i), 0, 1,
+ (String[])null));
+ }
+ return result;
+ }
+
+ /**
+ * Return a single record (filename, "") where the filename is taken from
+ * the file split.
+ */
+ static class RandomRecordReader extends RecordReader<Text, Text> {
+ Path name;
+ Text key = null;
+ Text value = new Text();
+ public RandomRecordReader(Path p) {
+ name = p;
+ }
+
+ public void initialize(InputSplit split,
+ TaskAttemptContext context)
+ throws IOException, InterruptedException {
+
+ }
+
+ public boolean nextKeyValue() {
+ if (name != null) {
+ key = new Text();
+ key.set(name.getName());
+ name = null;
+ return true;
+ }
+ return false;
+ }
+
+ public Text getCurrentKey() {
+ return key;
+ }
+
+ public Text getCurrentValue() {
+ return value;
+ }
+
+ public void close() {}
+
+ public float getProgress() {
+ return 0.0f;
+ }
+ }
+
+ public RecordReader<Text, Text> createRecordReader(InputSplit split,
+ TaskAttemptContext context) throws IOException, InterruptedException {
+ return new RandomRecordReader(((FileSplit) split).getPath());
+ }
+ }
+
+ static class RandomMapper extends Mapper<WritableComparable, Writable,
+ BytesWritable, BytesWritable> {
+
+ private long numBytesToWrite;
+ private int minKeySize;
+ private int keySizeRange;
+ private int minValueSize;
+ private int valueSizeRange;
+ private Random random = new Random();
+ private BytesWritable randomKey = new BytesWritable();
+ private BytesWritable randomValue = new BytesWritable();
+
+ private void randomizeBytes(byte[] data, int offset, int length) {
+ for(int i=offset + length - 1; i >= offset; --i) {
+ data[i] = (byte) random.nextInt(256);
+ }
+ }
+
+ /**
+ * Given an output filename, write a bunch of random records to it.
+ */
+ public void map(WritableComparable key,
+ Writable value,
+ Context context) throws IOException,InterruptedException {
+ int itemCount = 0;
+ while (numBytesToWrite > 0) {
+ int keyLength = minKeySize +
+ (keySizeRange != 0 ? random.nextInt(keySizeRange) : 0);
+ randomKey.setSize(keyLength);
+ randomizeBytes(randomKey.getBytes(), 0, randomKey.getLength());
+ int valueLength = minValueSize +
+ (valueSizeRange != 0 ? random.nextInt(valueSizeRange) : 0);
+ randomValue.setSize(valueLength);
+ randomizeBytes(randomValue.getBytes(), 0, randomValue.getLength());
+ context.write(randomKey, randomValue);
+ numBytesToWrite -= keyLength + valueLength;
+ context.getCounter(Counters.BYTES_WRITTEN).increment(keyLength + valueLength);
+ context.getCounter(Counters.RECORDS_WRITTEN).increment(1);
+ if (++itemCount % 200 == 0) {
+ context.setStatus("wrote record " + itemCount + ". " +
+ numBytesToWrite + " bytes left.");
+ }
+ }
+ context.setStatus("done with " + itemCount + " records.");
+ }
+
+ /**
+ * Save the values out of the configuaration that we need to write
+ * the data.
+ */
+ @Override
+ public void setup(Context context) {
+ Configuration conf = context.getConfiguration();
+ numBytesToWrite = conf.getLong(BYTES_PER_MAP,
+ 1*1024*1024*1024);
+ minKeySize = conf.getInt(MIN_KEY, 10);
+ keySizeRange =
+ conf.getInt(MAX_KEY, 1000) - minKeySize;
+ minValueSize = conf.getInt(MIN_VALUE, 0);
+ valueSizeRange =
+ conf.getInt(MAX_VALUE, 20000) - minValueSize;
+ }
+ }
+
+ /**
+ * This is the main routine for launching a distributed random write job.
+ * It runs 10 maps/node and each node writes 1 gig of data to a DFS file.
+ * The reduce doesn't do anything.
+ *
+ * @throws IOException
+ */
+ public int run(String[] args) throws Exception {
+ if (args.length == 0) {
+ System.out.println("Usage: writer <out-dir>");
+ ToolRunner.printGenericCommandUsage(System.out);
+ return 2;
+ }
+
+ Path outDir = new Path(args[0]);
+ Configuration conf = getConf();
+ JobClient client = new JobClient(conf);
+ ClusterStatus cluster = client.getClusterStatus();
+ int numMapsPerHost = conf.getInt(MAPS_PER_HOST, 10);
+ long numBytesToWritePerMap = conf.getLong(BYTES_PER_MAP,
+ 1*1024*1024*1024);
+ if (numBytesToWritePerMap == 0) {
+ System.err.println("Cannot have" + BYTES_PER_MAP + " set to 0");
+ return -2;
+ }
+ long totalBytesToWrite = conf.getLong(TOTAL_BYTES,
+ numMapsPerHost*numBytesToWritePerMap*cluster.getTaskTrackers());
+ int numMaps = (int) (totalBytesToWrite / numBytesToWritePerMap);
+ if (numMaps == 0 && totalBytesToWrite > 0) {
+ numMaps = 1;
+ conf.setLong(BYTES_PER_MAP, totalBytesToWrite);
+ }
+ conf.setInt(MRJobConfig.NUM_MAPS, numMaps);
+
+ Job job = new Job(conf);
+
+ job.setJarByClass(RandomWriter.class);
+ job.setJobName("random-writer");
+ FileOutputFormat.setOutputPath(job, outDir);
+ job.setOutputKeyClass(BytesWritable.class);
+ job.setOutputValueClass(BytesWritable.class);
+ job.setInputFormatClass(RandomInputFormat.class);
+ job.setMapperClass(RandomMapper.class);
+ job.setReducerClass(Reducer.class);
+ job.setOutputFormatClass(SequenceFileOutputFormat.class);
+
+ System.out.println("Running " + numMaps + " maps.");
+
+ // reducer NONE
+ job.setNumReduceTasks(0);
+
+ Date startTime = new Date();
+ System.out.println("Job started: " + startTime);
+ int ret = job.waitForCompletion(true) ? 0 : 1;
+ Date endTime = new Date();
+ System.out.println("Job ended: " + endTime);
+ System.out.println("The job took " +
+ (endTime.getTime() - startTime.getTime()) /1000 +
+ " seconds.");
+
+ return ret;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(), new RandomWriter(), args);
+ System.exit(res);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/SecondarySort.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/SecondarySort.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/SecondarySort.java
new file mode 100644
index 0000000..cdae905
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/SecondarySort.java
@@ -0,0 +1,248 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.StringTokenizer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Partitioner;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * This is an example Hadoop Map/Reduce application.
+ * It reads the text input files that must contain two integers per a line.
+ * The output is sorted by the first and second number and grouped on the
+ * first number.
+ *
+ * To run: bin/hadoop jar build/hadoop-examples.jar secondarysort
+ * <i>in-dir</i> <i>out-dir</i>
+ */
+public class SecondarySort extends Configured implements Tool {
+
+ /**
+ * Define a pair of integers that are writable.
+ * They are serialized in a byte comparable format.
+ */
+ public static class IntPair
+ implements WritableComparable<IntPair> {
+ private int first = 0;
+ private int second = 0;
+
+ /**
+ * Set the left and right values.
+ */
+ public void set(int left, int right) {
+ first = left;
+ second = right;
+ }
+ public int getFirst() {
+ return first;
+ }
+ public int getSecond() {
+ return second;
+ }
+ /**
+ * Read the two integers.
+ * Encoded as: MIN_VALUE -> 0, 0 -> -MIN_VALUE, MAX_VALUE-> -1
+ */
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ first = in.readInt() + Integer.MIN_VALUE;
+ second = in.readInt() + Integer.MIN_VALUE;
+ }
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(first - Integer.MIN_VALUE);
+ out.writeInt(second - Integer.MIN_VALUE);
+ }
+ @Override
+ public int hashCode() {
+ return first * 157 + second;
+ }
+ @Override
+ public boolean equals(Object right) {
+ if (right instanceof IntPair) {
+ IntPair r = (IntPair) right;
+ return r.first == first && r.second == second;
+ } else {
+ return false;
+ }
+ }
+ /** A Comparator that compares serialized IntPair. */
+ public static class Comparator extends WritableComparator {
+ public Comparator() {
+ super(IntPair.class);
+ }
+
+ public int compare(byte[] b1, int s1, int l1,
+ byte[] b2, int s2, int l2) {
+ return compareBytes(b1, s1, l1, b2, s2, l2);
+ }
+ }
+
+ static { // register this comparator
+ WritableComparator.define(IntPair.class, new Comparator());
+ }
+
+ @Override
+ public int compareTo(IntPair o) {
+ if (first != o.first) {
+ return first < o.first ? -1 : 1;
+ } else if (second != o.second) {
+ return second < o.second ? -1 : 1;
+ } else {
+ return 0;
+ }
+ }
+ }
+
+ /**
+ * Partition based on the first part of the pair.
+ */
+ public static class FirstPartitioner extends Partitioner<IntPair,IntWritable>{
+ @Override
+ public int getPartition(IntPair key, IntWritable value,
+ int numPartitions) {
+ return Math.abs(key.getFirst() * 127) % numPartitions;
+ }
+ }
+
+ /**
+ * Compare only the first part of the pair, so that reduce is called once
+ * for each value of the first part.
+ */
+ public static class FirstGroupingComparator
+ implements RawComparator<IntPair> {
+ @Override
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+ return WritableComparator.compareBytes(b1, s1, Integer.SIZE/8,
+ b2, s2, Integer.SIZE/8);
+ }
+
+ @Override
+ public int compare(IntPair o1, IntPair o2) {
+ int l = o1.getFirst();
+ int r = o2.getFirst();
+ return l == r ? 0 : (l < r ? -1 : 1);
+ }
+ }
+
+ /**
+ * Read two integers from each line and generate a key, value pair
+ * as ((left, right), right).
+ */
+ public static class MapClass
+ extends Mapper<LongWritable, Text, IntPair, IntWritable> {
+
+ private final IntPair key = new IntPair();
+ private final IntWritable value = new IntWritable();
+
+ @Override
+ public void map(LongWritable inKey, Text inValue,
+ Context context) throws IOException, InterruptedException {
+ StringTokenizer itr = new StringTokenizer(inValue.toString());
+ int left = 0;
+ int right = 0;
+ if (itr.hasMoreTokens()) {
+ left = Integer.parseInt(itr.nextToken());
+ if (itr.hasMoreTokens()) {
+ right = Integer.parseInt(itr.nextToken());
+ }
+ key.set(left, right);
+ value.set(right);
+ context.write(key, value);
+ }
+ }
+ }
+
+ /**
+ * A reducer class that just emits the sum of the input values.
+ */
+ public static class Reduce
+ extends Reducer<IntPair, IntWritable, Text, IntWritable> {
+ private static final Text SEPARATOR =
+ new Text("------------------------------------------------");
+ private final Text first = new Text();
+
+ @Override
+ public void reduce(IntPair key, Iterable<IntWritable> values,
+ Context context
+ ) throws IOException, InterruptedException {
+ context.write(SEPARATOR, null);
+ first.set(Integer.toString(key.getFirst()));
+ for(IntWritable value: values) {
+ context.write(first, value);
+ }
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Configuration conf = getConf();
+ String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+ if (otherArgs.length != 2) {
+ System.err.println("Usage: secondarysort <in> <out>");
+ ToolRunner.printGenericCommandUsage(System.out);
+ System.exit(2);
+ }
+ Job job = new Job(conf, "secondary sort");
+ job.setJarByClass(SecondarySort.class);
+ job.setMapperClass(MapClass.class);
+ job.setReducerClass(Reduce.class);
+
+ // group and partition by the first int in the pair
+ job.setPartitionerClass(FirstPartitioner.class);
+ job.setGroupingComparatorClass(FirstGroupingComparator.class);
+
+ // the map output is IntPair, IntWritable
+ job.setMapOutputKeyClass(IntPair.class);
+ job.setMapOutputValueClass(IntWritable.class);
+
+ // the reduce output is Text, IntWritable
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+
+ FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
+ FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
+ return job.waitForCompletion(true) ? 0 : 1;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(), new SecondarySort(), args);
+ System.exit(res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Sort.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Sort.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Sort.java
new file mode 100644
index 0000000..13e1bbc
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Sort.java
@@ -0,0 +1,203 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples;
+
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.mapreduce.filecache.DistributedCache;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapred.ClusterStatus;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.partition.InputSampler;
+import org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * This is the trivial map/reduce program that does absolutely nothing
+ * other than use the framework to fragment and sort the input values.
+ *
+ * To run: bin/hadoop jar build/hadoop-examples.jar sort
+ * [-r <i>reduces</i>]
+ * [-inFormat <i>input format class</i>]
+ * [-outFormat <i>output format class</i>]
+ * [-outKey <i>output key class</i>]
+ * [-outValue <i>output value class</i>]
+ * [-totalOrder <i>pcnt</i> <i>num samples</i> <i>max splits</i>]
+ * <i>in-dir</i> <i>out-dir</i>
+ */
+public class Sort<K,V> extends Configured implements Tool {
+ public static final String REDUCES_PER_HOST =
+ "mapreduce.sort.reducesperhost";
+ private Job job = null;
+
+ static int printUsage() {
+ System.out.println("sort [-r <reduces>] " +
+ "[-inFormat <input format class>] " +
+ "[-outFormat <output format class>] " +
+ "[-outKey <output key class>] " +
+ "[-outValue <output value class>] " +
+ "[-totalOrder <pcnt> <num samples> <max splits>] " +
+ "<input> <output>");
+ ToolRunner.printGenericCommandUsage(System.out);
+ return 2;
+ }
+
+ /**
+ * The main driver for sort program.
+ * Invoke this method to submit the map/reduce job.
+ * @throws java.lang.Exception When there is communication problems with the
+ * job tracker.
+ */
+ public int run(String[] args) throws Exception {
+
+ Configuration conf = getConf();
+ JobClient client = new JobClient(conf);
+ ClusterStatus cluster = client.getClusterStatus();
+ int num_reduces = (int) (cluster.getMaxReduceTasks() * 0.9);
+ String sort_reduces = conf.get(REDUCES_PER_HOST);
+ if (sort_reduces != null) {
+ num_reduces = cluster.getTaskTrackers() *
+ Integer.parseInt(sort_reduces);
+ }
+ Class<? extends InputFormat> inputFormatClass =
+ SequenceFileInputFormat.class;
+ Class<? extends OutputFormat> outputFormatClass =
+ SequenceFileOutputFormat.class;
+ Class<? extends WritableComparable> outputKeyClass = BytesWritable.class;
+ Class<? extends Writable> outputValueClass = BytesWritable.class;
+ List<String> otherArgs = new ArrayList<String>();
+ InputSampler.Sampler<K,V> sampler = null;
+ for(int i=0; i < args.length; ++i) {
+ try {
+ if ("-r".equals(args[i])) {
+ num_reduces = Integer.parseInt(args[++i]);
+ } else if ("-inFormat".equals(args[i])) {
+ inputFormatClass =
+ Class.forName(args[++i]).asSubclass(InputFormat.class);
+ } else if ("-outFormat".equals(args[i])) {
+ outputFormatClass =
+ Class.forName(args[++i]).asSubclass(OutputFormat.class);
+ } else if ("-outKey".equals(args[i])) {
+ outputKeyClass =
+ Class.forName(args[++i]).asSubclass(WritableComparable.class);
+ } else if ("-outValue".equals(args[i])) {
+ outputValueClass =
+ Class.forName(args[++i]).asSubclass(Writable.class);
+ } else if ("-totalOrder".equals(args[i])) {
+ double pcnt = Double.parseDouble(args[++i]);
+ int numSamples = Integer.parseInt(args[++i]);
+ int maxSplits = Integer.parseInt(args[++i]);
+ if (0 >= maxSplits) maxSplits = Integer.MAX_VALUE;
+ sampler =
+ new InputSampler.RandomSampler<K,V>(pcnt, numSamples, maxSplits);
+ } else {
+ otherArgs.add(args[i]);
+ }
+ } catch (NumberFormatException except) {
+ System.out.println("ERROR: Integer expected instead of " + args[i]);
+ return printUsage();
+ } catch (ArrayIndexOutOfBoundsException except) {
+ System.out.println("ERROR: Required parameter missing from " +
+ args[i-1]);
+ return printUsage(); // exits
+ }
+ }
+ // Set user-supplied (possibly default) job configs
+ job = new Job(conf);
+ job.setJobName("sorter");
+ job.setJarByClass(Sort.class);
+
+ job.setMapperClass(Mapper.class);
+ job.setReducerClass(Reducer.class);
+
+ job.setNumReduceTasks(num_reduces);
+
+ job.setInputFormatClass(inputFormatClass);
+ job.setOutputFormatClass(outputFormatClass);
+
+ job.setOutputKeyClass(outputKeyClass);
+ job.setOutputValueClass(outputValueClass);
+
+ // Make sure there are exactly 2 parameters left.
+ if (otherArgs.size() != 2) {
+ System.out.println("ERROR: Wrong number of parameters: " +
+ otherArgs.size() + " instead of 2.");
+ return printUsage();
+ }
+ FileInputFormat.setInputPaths(job, otherArgs.get(0));
+ FileOutputFormat.setOutputPath(job, new Path(otherArgs.get(1)));
+
+ if (sampler != null) {
+ System.out.println("Sampling input to effect total-order sort...");
+ job.setPartitionerClass(TotalOrderPartitioner.class);
+ Path inputDir = FileInputFormat.getInputPaths(job)[0];
+ inputDir = inputDir.makeQualified(inputDir.getFileSystem(conf));
+ Path partitionFile = new Path(inputDir, "_sortPartitioning");
+ TotalOrderPartitioner.setPartitionFile(conf, partitionFile);
+ InputSampler.<K,V>writePartitionFile(job, sampler);
+ URI partitionUri = new URI(partitionFile.toString() +
+ "#" + "_sortPartitioning");
+ DistributedCache.addCacheFile(partitionUri, conf);
+ }
+
+ System.out.println("Running on " +
+ cluster.getTaskTrackers() +
+ " nodes to sort from " +
+ FileInputFormat.getInputPaths(job)[0] + " into " +
+ FileOutputFormat.getOutputPath(job) +
+ " with " + num_reduces + " reduces.");
+ Date startTime = new Date();
+ System.out.println("Job started: " + startTime);
+ int ret = job.waitForCompletion(true) ? 0 : 1;
+ Date end_time = new Date();
+ System.out.println("Job ended: " + end_time);
+ System.out.println("The job took " +
+ (end_time.getTime() - startTime.getTime()) /1000 + " seconds.");
+ return ret;
+ }
+
+
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(), new Sort(), args);
+ System.exit(res);
+ }
+
+ /**
+ * Get the last job that was run using this instance.
+ * @return the results of the last job that was run
+ */
+ public Job getResult() {
+ return job;
+ }
+}
[08/10] TEZ-1055. Rename tez-mapreduce-examples to tez-examples
(Hitesh Shah via bikas)
Posted by bi...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/Join.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/Join.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/Join.java
deleted file mode 100644
index a3a65ec..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/Join.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples;
-
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.mapred.ClusterStatus;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.join.CompositeInputFormat;
-import org.apache.hadoop.mapreduce.lib.join.TupleWritable;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-/**
- * Given a set of sorted datasets keyed with the same class and yielding
- * equal partitions, it is possible to effect a join of those datasets
- * prior to the map. The example facilitates the same.
- *
- * To run: bin/hadoop jar build/hadoop-examples.jar join
- * [-r <i>reduces</i>]
- * [-inFormat <i>input format class</i>]
- * [-outFormat <i>output format class</i>]
- * [-outKey <i>output key class</i>]
- * [-outValue <i>output value class</i>]
- * [-joinOp <inner|outer|override>]
- * [<i>in-dir</i>]* <i>in-dir</i> <i>out-dir</i>
- */
-public class Join extends Configured implements Tool {
- public static final String REDUCES_PER_HOST = "mapreduce.join.reduces_per_host";
- static int printUsage() {
- System.out.println("join [-r <reduces>] " +
- "[-inFormat <input format class>] " +
- "[-outFormat <output format class>] " +
- "[-outKey <output key class>] " +
- "[-outValue <output value class>] " +
- "[-joinOp <inner|outer|override>] " +
- "[input]* <input> <output>");
- ToolRunner.printGenericCommandUsage(System.out);
- return 2;
- }
-
- /**
- * The main driver for sort program.
- * Invoke this method to submit the map/reduce job.
- * @throws IOException When there is communication problems with the
- * job tracker.
- */
- @SuppressWarnings("unchecked")
- public int run(String[] args) throws Exception {
- Configuration conf = getConf();
- JobClient client = new JobClient(conf);
- ClusterStatus cluster = client.getClusterStatus();
- int num_reduces = (int) (cluster.getMaxReduceTasks() * 0.9);
- String join_reduces = conf.get(REDUCES_PER_HOST);
- if (join_reduces != null) {
- num_reduces = cluster.getTaskTrackers() *
- Integer.parseInt(join_reduces);
- }
- Job job = new Job(conf);
- job.setJobName("join");
- job.setJarByClass(Sort.class);
-
- job.setMapperClass(Mapper.class);
- job.setReducerClass(Reducer.class);
-
- Class<? extends InputFormat> inputFormatClass =
- SequenceFileInputFormat.class;
- Class<? extends OutputFormat> outputFormatClass =
- SequenceFileOutputFormat.class;
- Class<? extends WritableComparable> outputKeyClass = BytesWritable.class;
- Class<? extends Writable> outputValueClass = TupleWritable.class;
- String op = "inner";
- List<String> otherArgs = new ArrayList<String>();
- for(int i=0; i < args.length; ++i) {
- try {
- if ("-r".equals(args[i])) {
- num_reduces = Integer.parseInt(args[++i]);
- } else if ("-inFormat".equals(args[i])) {
- inputFormatClass =
- Class.forName(args[++i]).asSubclass(InputFormat.class);
- } else if ("-outFormat".equals(args[i])) {
- outputFormatClass =
- Class.forName(args[++i]).asSubclass(OutputFormat.class);
- } else if ("-outKey".equals(args[i])) {
- outputKeyClass =
- Class.forName(args[++i]).asSubclass(WritableComparable.class);
- } else if ("-outValue".equals(args[i])) {
- outputValueClass =
- Class.forName(args[++i]).asSubclass(Writable.class);
- } else if ("-joinOp".equals(args[i])) {
- op = args[++i];
- } else {
- otherArgs.add(args[i]);
- }
- } catch (NumberFormatException except) {
- System.out.println("ERROR: Integer expected instead of " + args[i]);
- return printUsage();
- } catch (ArrayIndexOutOfBoundsException except) {
- System.out.println("ERROR: Required parameter missing from " +
- args[i-1]);
- return printUsage(); // exits
- }
- }
-
- // Set user-supplied (possibly default) job configs
- job.setNumReduceTasks(num_reduces);
-
- if (otherArgs.size() < 2) {
- System.out.println("ERROR: Wrong number of parameters: ");
- return printUsage();
- }
-
- FileOutputFormat.setOutputPath(job,
- new Path(otherArgs.remove(otherArgs.size() - 1)));
- List<Path> plist = new ArrayList<Path>(otherArgs.size());
- for (String s : otherArgs) {
- plist.add(new Path(s));
- }
-
- job.setInputFormatClass(CompositeInputFormat.class);
- job.getConfiguration().set(CompositeInputFormat.JOIN_EXPR,
- CompositeInputFormat.compose(op, inputFormatClass,
- plist.toArray(new Path[0])));
- job.setOutputFormatClass(outputFormatClass);
-
- job.setOutputKeyClass(outputKeyClass);
- job.setOutputValueClass(outputValueClass);
-
- Date startTime = new Date();
- System.out.println("Job started: " + startTime);
- int ret = job.waitForCompletion(true) ? 0 : 1 ;
- Date end_time = new Date();
- System.out.println("Job ended: " + end_time);
- System.out.println("The job took " +
- (end_time.getTime() - startTime.getTime()) /1000 + " seconds.");
- return ret;
- }
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new Join(), args);
- System.exit(res);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
deleted file mode 100644
index b22a4a8..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
+++ /dev/null
@@ -1,757 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.tez.mapreduce.examples;
-
-import java.io.IOException;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.google.common.collect.Maps;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Partitioner;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
-import org.apache.hadoop.mapreduce.security.TokenCache;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.util.ClassUtil;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.tez.client.TezClientUtils;
-import org.apache.tez.client.TezClient;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.DataSourceDescriptor;
-import org.apache.tez.dag.api.Edge;
-import org.apache.tez.dag.api.ProcessorDescriptor;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.api.Vertex;
-import org.apache.tez.dag.api.client.DAGClient;
-import org.apache.tez.dag.api.client.DAGStatus;
-import org.apache.tez.mapreduce.hadoop.MRHelpers;
-import org.apache.tez.mapreduce.hadoop.MRInputHelpers;
-import org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil;
-import org.apache.tez.mapreduce.input.MRInputLegacy;
-import org.apache.tez.mapreduce.output.MROutputLegacy;
-import org.apache.tez.mapreduce.processor.map.MapProcessor;
-import org.apache.tez.mapreduce.processor.reduce.ReduceProcessor;
-import org.apache.tez.runtime.library.conf.OrderedPartitionedKVEdgeConfigurer;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.tez.runtime.library.partitioner.HashPartitioner;
-
-/**
- * Dummy class for testing MR framefork. Sleeps for a defined period
- * of time in mapper and reducer. Generates fake input for map / reduce
- * jobs. Note that generated number of input pairs is in the order
- * of <code>numMappers * mapSleepTime / 100</code>, so the job uses
- * some disk space.
- */
-public class MRRSleepJob extends Configured implements Tool {
-
- private static final Log LOG = LogFactory.getLog(MRRSleepJob.class);
-
- public static final String MAP_SLEEP_COUNT = "mrr.sleepjob.map.sleep.count";
- public static final String REDUCE_SLEEP_COUNT =
- "mrr.sleepjob.reduce.sleep.count";
- public static final String MAP_SLEEP_TIME = "mrr.sleepjob.map.sleep.time";
- public static final String REDUCE_SLEEP_TIME =
- "mrr.sleepjob.reduce.sleep.time";
- public static final String IREDUCE_SLEEP_COUNT =
- "mrr.sleepjob.ireduce.sleep.count";
- public static final String IREDUCE_SLEEP_TIME =
- "mrr.sleepjob.ireduce.sleep.time";
- public static final String IREDUCE_STAGES_COUNT =
- "mrr.sleepjob.ireduces.stages.count";
- public static final String IREDUCE_TASKS_COUNT =
- "mrr.sleepjob.ireduces.tasks.count";
-
- // Flags to inject failures
- public static final String MAP_THROW_ERROR = "mrr.sleepjob.map.throw.error";
- public static final String MAP_FATAL_ERROR = "mrr.sleepjob.map.fatal.error";
- public static final String MAP_ERROR_TASK_IDS =
- "mrr.sleepjob.map.error.task.ids";
-
- public static class MRRSleepJobPartitioner extends
- Partitioner<IntWritable, IntWritable> {
- public int getPartition(IntWritable k, IntWritable v, int numPartitions) {
- return k.get() % numPartitions;
- }
- }
-
- public static class EmptySplit extends InputSplit implements Writable {
- public void write(DataOutput out) throws IOException { }
- public void readFields(DataInput in) throws IOException { }
- public long getLength() { return 0L; }
- public String[] getLocations() { return new String[0]; }
- }
-
- public static class SleepInputFormat
- extends InputFormat<IntWritable,IntWritable> {
-
- public List<InputSplit> getSplits(JobContext jobContext) {
- List<InputSplit> ret = new ArrayList<InputSplit>();
- int numSplits = jobContext.getConfiguration().
- getInt(MRJobConfig.NUM_MAPS, 1);
- for (int i = 0; i < numSplits; ++i) {
- ret.add(new EmptySplit());
- }
- return ret;
- }
-
- public RecordReader<IntWritable,IntWritable> createRecordReader(
- InputSplit ignored, TaskAttemptContext taskContext)
- throws IOException {
- Configuration conf = taskContext.getConfiguration();
-
- final int count = conf.getInt(MAP_SLEEP_COUNT, 1);
- if (count < 0) {
- throw new IOException("Invalid map count: " + count);
- }
-
- int totalIReduces = conf.getInt(IREDUCE_STAGES_COUNT, 1);
-
- int reduceTasks = totalIReduces == 0?
- taskContext.getNumReduceTasks() :
- conf.getInt(IREDUCE_TASKS_COUNT, 1);
- int sleepCount = totalIReduces == 0?
- conf.getInt(REDUCE_SLEEP_COUNT,1) :
- conf.getInt(IREDUCE_SLEEP_COUNT,1);
- final int emitPerMapTask = sleepCount * reduceTasks;
-
- return new RecordReader<IntWritable,IntWritable>() {
- private int records = 0;
- private int emitCount = 0;
- private IntWritable key = null;
- private IntWritable value = null;
-
- public void initialize(InputSplit split, TaskAttemptContext context) {
- }
-
- public boolean nextKeyValue()
- throws IOException {
- if (count == 0) {
- return false;
- }
- key = new IntWritable();
- key.set(emitCount);
- int emit = emitPerMapTask / count;
- if ((emitPerMapTask) % count > records) {
- ++emit;
- }
- emitCount += emit;
- value = new IntWritable();
- value.set(emit);
- return records++ < count;
- }
- public IntWritable getCurrentKey() { return key; }
- public IntWritable getCurrentValue() { return value; }
- public void close() throws IOException { }
- public float getProgress() throws IOException {
- return count == 0 ? 100 : records / ((float)count);
- }
- };
- }
- }
-
- public static class SleepMapper
- extends Mapper<IntWritable, IntWritable, IntWritable, IntWritable> {
- private long mapSleepDuration = 100;
- private int mapSleepCount = 1;
- private int count = 0;
- private String vertexName;
- private boolean throwError = false;
- private boolean throwFatal = false;
- private boolean finalAttempt = false;
-
- protected void setup(Context context)
- throws IOException, InterruptedException {
- Configuration conf = context.getConfiguration();
- this.mapSleepCount =
- conf.getInt(MAP_SLEEP_COUNT, mapSleepCount);
- this.mapSleepDuration = mapSleepCount == 0 ? 0 :
- conf.getLong(MAP_SLEEP_TIME , 100) / mapSleepCount;
- vertexName = conf.get(
- org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);
-
- TaskAttemptID taId = context.getTaskAttemptID();
-
- String[] taskIds = conf.getStrings(MAP_ERROR_TASK_IDS);
- if (taId.getId()+1 >= context.getMaxMapAttempts()) {
- finalAttempt = true;
- }
- boolean found = false;
- if (taskIds != null) {
- if (taskIds.length == 1 && taskIds[0].equals("*")) {
- found = true;
- }
- if (!found) {
- for (String taskId : taskIds) {
- if (Integer.valueOf(taskId).intValue() ==
- taId.getTaskID().getId()) {
- found = true;
- break;
- }
- }
- }
- }
- if (found) {
- if (!finalAttempt) {
- throwError = conf.getBoolean(MAP_THROW_ERROR, false);
- }
- throwFatal = conf.getBoolean(MAP_FATAL_ERROR, false);
- }
- }
-
- public void map(IntWritable key, IntWritable value, Context context
- ) throws IOException, InterruptedException {
- //it is expected that every map processes mapSleepCount number of records.
- try {
- LOG.info("Reading in " + vertexName
- + " taskid " + context.getTaskAttemptID().getTaskID().getId()
- + " key " + key.get());
- LOG.info("Sleeping in InitialMap"
- + ", vertexName=" + vertexName
- + ", taskAttemptId=" + context.getTaskAttemptID()
- + ", mapSleepDuration=" + mapSleepDuration
- + ", mapSleepCount=" + mapSleepCount
- + ", sleepLeft="
- + (mapSleepDuration * (mapSleepCount - count)));
- context.setStatus("Sleeping... (" +
- (mapSleepDuration * (mapSleepCount - count)) + ") ms left");
- if ((mapSleepCount - count) > 0) {
- Thread.sleep(mapSleepDuration);
- }
- if (throwError || throwFatal) {
- throw new IOException("Throwing a simulated error from map");
- }
- }
- catch (InterruptedException ex) {
- throw (IOException)new IOException(
- "Interrupted while sleeping").initCause(ex);
- }
- ++count;
- // output reduceSleepCount * numReduce number of random values, so that
- // each reducer will get reduceSleepCount number of keys.
- int k = key.get();
- for (int i = 0; i < value.get(); ++i) {
- LOG.info("Writing in " + vertexName
- + " taskid " + context.getTaskAttemptID().getTaskID().getId()
- + " key " + (k+i) + " value 1");
- context.write(new IntWritable(k + i), new IntWritable(1));
- }
- }
- }
-
- public static class ISleepReducer
- extends Reducer<IntWritable, IntWritable, IntWritable, IntWritable> {
- private long iReduceSleepDuration = 100;
- private int iReduceSleepCount = 1;
- private int count = 0;
- private String vertexName;
-
- protected void setup(Context context)
- throws IOException, InterruptedException {
- Configuration conf = context.getConfiguration();
- this.iReduceSleepCount =
- conf.getInt(IREDUCE_SLEEP_COUNT, iReduceSleepCount);
- this.iReduceSleepDuration = iReduceSleepCount == 0 ? 0 :
- conf.getLong(IREDUCE_SLEEP_TIME , 100) / iReduceSleepCount;
- vertexName = conf.get(
- org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);
- }
-
- public void reduce(IntWritable key, Iterable<IntWritable> values,
- Context context)
- throws IOException, InterruptedException {
- try {
- LOG.info("Reading in " + vertexName
- + " taskid " + context.getTaskAttemptID().getTaskID().getId()
- + " key " + key.get());
-
- LOG.info("Sleeping in IntermediateReduce"
- + ", vertexName=" + vertexName
- + ", taskAttemptId=" + context.getTaskAttemptID()
- + ", iReduceSleepDuration=" + iReduceSleepDuration
- + ", iReduceSleepCount=" + iReduceSleepCount
- + ", sleepLeft="
- + (iReduceSleepDuration * (iReduceSleepCount - count)));
- context.setStatus("Sleeping... (" +
- (iReduceSleepDuration * (iReduceSleepCount - count)) + ") ms left");
- if ((iReduceSleepCount - count) > 0) {
- Thread.sleep(iReduceSleepDuration);
- }
- }
- catch (InterruptedException ex) {
- throw (IOException)new IOException(
- "Interrupted while sleeping").initCause(ex);
- }
- ++count;
- // output reduceSleepCount * numReduce number of random values, so that
- // each reducer will get reduceSleepCount number of keys.
- int k = key.get();
- for (IntWritable value : values) {
- for (int i = 0; i < value.get(); ++i) {
- LOG.info("Writing in " + vertexName
- + " taskid " + context.getTaskAttemptID().getTaskID().getId()
- + " key " + (k+i) + " value 1");
- context.write(new IntWritable(k + i), new IntWritable(1));
- }
- }
- }
- }
-
- public static class SleepReducer
- extends Reducer<IntWritable, IntWritable, NullWritable, NullWritable> {
- private long reduceSleepDuration = 100;
- private int reduceSleepCount = 1;
- private int count = 0;
- private String vertexName;
-
- protected void setup(Context context)
- throws IOException, InterruptedException {
- Configuration conf = context.getConfiguration();
- this.reduceSleepCount =
- conf.getInt(REDUCE_SLEEP_COUNT, reduceSleepCount);
- this.reduceSleepDuration = reduceSleepCount == 0 ? 0 :
- conf.getLong(REDUCE_SLEEP_TIME , 100) / reduceSleepCount;
- vertexName = conf.get(
- org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);
- }
-
- public void reduce(IntWritable key, Iterable<IntWritable> values,
- Context context)
- throws IOException {
- try {
- LOG.info("Reading in " + vertexName
- + " taskid " + context.getTaskAttemptID().getTaskID().getId()
- + " key " + key.get());
- LOG.info("Sleeping in FinalReduce"
- + ", vertexName=" + vertexName
- + ", taskAttemptId=" + context.getTaskAttemptID()
- + ", reduceSleepDuration=" + reduceSleepDuration
- + ", reduceSleepCount=" + reduceSleepCount
- + ", sleepLeft="
- + (reduceSleepDuration * (reduceSleepCount - count)));
- context.setStatus("Sleeping... (" +
- (reduceSleepDuration * (reduceSleepCount - count)) + ") ms left");
- if ((reduceSleepCount - count) > 0) {
- Thread.sleep(reduceSleepDuration);
- }
- }
- catch (InterruptedException ex) {
- throw (IOException)new IOException(
- "Interrupted while sleeping").initCause(ex);
- }
- count++;
- }
- }
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new MRRSleepJob(), args);
- System.exit(res);
- }
-
- private Credentials credentials = new Credentials();
-
- public DAG createDAG(FileSystem remoteFs, Configuration conf, Path remoteStagingDir,
- int numMapper, int numReducer, int iReduceStagesCount,
- int numIReducer, long mapSleepTime, int mapSleepCount,
- long reduceSleepTime, int reduceSleepCount,
- long iReduceSleepTime, int iReduceSleepCount, boolean writeSplitsToDFS,
- boolean generateSplitsInAM)
- throws IOException, YarnException {
-
-
- Configuration mapStageConf = new JobConf(conf);
- mapStageConf.setInt(MRJobConfig.NUM_MAPS, numMapper);
- mapStageConf.setLong(MAP_SLEEP_TIME, mapSleepTime);
- mapStageConf.setLong(REDUCE_SLEEP_TIME, reduceSleepTime);
- mapStageConf.setLong(IREDUCE_SLEEP_TIME, iReduceSleepTime);
- mapStageConf.setInt(MAP_SLEEP_COUNT, mapSleepCount);
- mapStageConf.setInt(REDUCE_SLEEP_COUNT, reduceSleepCount);
- mapStageConf.setInt(IREDUCE_SLEEP_COUNT, iReduceSleepCount);
- mapStageConf.setInt(IREDUCE_STAGES_COUNT, iReduceStagesCount);
- mapStageConf.setInt(IREDUCE_TASKS_COUNT, numIReducer);
- mapStageConf.set(MRJobConfig.MAP_CLASS_ATTR, SleepMapper.class.getName());
- mapStageConf.set(MRJobConfig.INPUT_FORMAT_CLASS_ATTR,
- SleepInputFormat.class.getName());
- if (numIReducer == 0 && numReducer == 0) {
- mapStageConf.set(MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR,
- NullOutputFormat.class.getName());
- }
-
- MRHelpers.translateMRConfToTez(mapStageConf);
-
- Configuration[] intermediateReduceStageConfs = null;
- if (iReduceStagesCount > 0
- && numIReducer > 0) {
- intermediateReduceStageConfs = new JobConf[iReduceStagesCount];
- for (int i = 1; i <= iReduceStagesCount; ++i) {
- JobConf iReduceStageConf = new JobConf(conf);
- iReduceStageConf.setLong(MRRSleepJob.REDUCE_SLEEP_TIME, iReduceSleepTime);
- iReduceStageConf.setInt(MRRSleepJob.REDUCE_SLEEP_COUNT, iReduceSleepCount);
- iReduceStageConf.setInt(MRJobConfig.NUM_REDUCES, numIReducer);
- iReduceStageConf
- .set(MRJobConfig.REDUCE_CLASS_ATTR, ISleepReducer.class.getName());
- iReduceStageConf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS,
- IntWritable.class.getName());
- iReduceStageConf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS,
- IntWritable.class.getName());
- iReduceStageConf.set(MRJobConfig.PARTITIONER_CLASS_ATTR,
- MRRSleepJobPartitioner.class.getName());
-
-
- MRHelpers.translateMRConfToTez(iReduceStageConf);
- intermediateReduceStageConfs[i-1] = iReduceStageConf;
- }
- }
-
- Configuration finalReduceConf = null;
- if (numReducer > 0) {
- finalReduceConf = new JobConf(conf);
- finalReduceConf.setLong(MRRSleepJob.REDUCE_SLEEP_TIME, reduceSleepTime);
- finalReduceConf.setInt(MRRSleepJob.REDUCE_SLEEP_COUNT, reduceSleepCount);
- finalReduceConf.setInt(MRJobConfig.NUM_REDUCES, numReducer);
- finalReduceConf.set(MRJobConfig.REDUCE_CLASS_ATTR, SleepReducer.class.getName());
- finalReduceConf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS,
- IntWritable.class.getName());
- finalReduceConf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS,
- IntWritable.class.getName());
- finalReduceConf.set(MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR,
- NullOutputFormat.class.getName());
-
- MRHelpers.translateMRConfToTez(finalReduceConf);
- }
-
- MRHelpers.configureMRApiUsage(mapStageConf);
- if (iReduceStagesCount > 0
- && numIReducer > 0) {
- for (int i = 0; i < iReduceStagesCount; ++i) {
- MRHelpers.configureMRApiUsage(intermediateReduceStageConfs[i]);
- }
- }
- if (numReducer > 0) {
- MRHelpers.configureMRApiUsage(finalReduceConf);
- }
-
- DataSourceDescriptor dataSource = null;
- if (!generateSplitsInAM && writeSplitsToDFS) {
-
- LOG.info("Writing splits to DFS");
- dataSource = MRInputHelpers
- .configureMRInputWithLegacySplitGeneration(mapStageConf, remoteStagingDir, true);
- } else {
- dataSource = MRInputLegacy.createConfigurer(mapStageConf, SleepInputFormat.class)
- .generateSplitsInAM(generateSplitsInAM).create();
- }
-
- DAG dag = new DAG("MRRSleepJob");
- String jarPath = ClassUtil.findContainingJar(getClass());
- if (jarPath == null) {
- throw new TezUncheckedException("Could not find any jar containing"
- + " MRRSleepJob.class in the classpath");
- }
- Path remoteJarPath = remoteFs.makeQualified(
- new Path(remoteStagingDir, "dag_job.jar"));
- remoteFs.copyFromLocalFile(new Path(jarPath), remoteJarPath);
- FileStatus jarFileStatus = remoteFs.getFileStatus(remoteJarPath);
-
- TokenCache.obtainTokensForNamenodes(this.credentials, new Path[] { remoteJarPath },
- mapStageConf);
-
- Map<String, LocalResource> commonLocalResources =
- new HashMap<String, LocalResource>();
- LocalResource dagJarLocalRsrc = LocalResource.newInstance(
- ConverterUtils.getYarnUrlFromPath(remoteJarPath),
- LocalResourceType.FILE,
- LocalResourceVisibility.APPLICATION,
- jarFileStatus.getLen(),
- jarFileStatus.getModificationTime());
- commonLocalResources.put("dag_job.jar", dagJarLocalRsrc);
-
- List<Vertex> vertices = new ArrayList<Vertex>();
-
-
- UserPayload mapUserPayload = TezUtils.createUserPayloadFromConf(mapStageConf);
- int numTasks = generateSplitsInAM ? -1 : numMapper;
-
- Vertex mapVertex = new Vertex("map", new ProcessorDescriptor(
- MapProcessor.class.getName()).setUserPayload(mapUserPayload), numTasks)
- .setTaskLocalFiles(commonLocalResources);
- mapVertex.addDataSource("MRInput", dataSource);
- vertices.add(mapVertex);
-
- if (iReduceStagesCount > 0
- && numIReducer > 0) {
- for (int i = 0; i < iReduceStagesCount; ++i) {
- Configuration iconf =
- intermediateReduceStageConfs[i];
- UserPayload iReduceUserPayload = TezUtils.createUserPayloadFromConf(iconf);
- Vertex ivertex = new Vertex("ireduce" + (i+1),
- new ProcessorDescriptor(ReduceProcessor.class.getName()).
- setUserPayload(iReduceUserPayload), numIReducer);
- ivertex.setTaskLocalFiles(commonLocalResources);
- vertices.add(ivertex);
- }
- }
-
- Vertex finalReduceVertex = null;
- if (numReducer > 0) {
- UserPayload reducePayload = TezUtils.createUserPayloadFromConf(finalReduceConf);
- finalReduceVertex = new Vertex("reduce", new ProcessorDescriptor(
- ReduceProcessor.class.getName()).setUserPayload(reducePayload), numReducer);
- finalReduceVertex.setTaskLocalFiles(commonLocalResources);
- finalReduceVertex.addDataSink("MROutput", MROutputLegacy.createConfigurer(finalReduceConf,
- NullOutputFormat.class).create());
- vertices.add(finalReduceVertex);
- } else {
- // Map only job
- mapVertex.addDataSink("MROutput",
- MROutputLegacy.createConfigurer(mapStageConf, NullOutputFormat.class).create());
- }
-
-
- Map<String, String> partitionerConf = Maps.newHashMap();
- partitionerConf.put(MRJobConfig.PARTITIONER_CLASS_ATTR, MRRSleepJobPartitioner.class.getName());
- OrderedPartitionedKVEdgeConfigurer edgeConf = OrderedPartitionedKVEdgeConfigurer
- .newBuilder(IntWritable.class.getName(), IntWritable.class.getName(),
- HashPartitioner.class.getName(), partitionerConf).configureInput().useLegacyInput()
- .done().build();
-
- for (int i = 0; i < vertices.size(); ++i) {
- dag.addVertex(vertices.get(i));
- if (i != 0) {
- dag.addEdge(
- new Edge(vertices.get(i - 1), vertices.get(i), edgeConf.createDefaultEdgeProperty()));
- }
- }
-
- return dag;
- }
-
- @VisibleForTesting
- public Job createJob(int numMapper, int numReducer, int iReduceStagesCount,
- int numIReducer, long mapSleepTime, int mapSleepCount,
- long reduceSleepTime, int reduceSleepCount,
- long iReduceSleepTime, int iReduceSleepCount)
- throws IOException {
- Configuration conf = getConf();
- conf.setLong(MAP_SLEEP_TIME, mapSleepTime);
- conf.setLong(REDUCE_SLEEP_TIME, reduceSleepTime);
- conf.setLong(IREDUCE_SLEEP_TIME, iReduceSleepTime);
- conf.setInt(MAP_SLEEP_COUNT, mapSleepCount);
- conf.setInt(REDUCE_SLEEP_COUNT, reduceSleepCount);
- conf.setInt(IREDUCE_SLEEP_COUNT, iReduceSleepCount);
- conf.setInt(MRJobConfig.NUM_MAPS, numMapper);
- conf.setInt(IREDUCE_STAGES_COUNT, iReduceStagesCount);
- conf.setInt(IREDUCE_TASKS_COUNT, numIReducer);
-
- // Configure intermediate reduces
- conf.setInt(
- org.apache.tez.mapreduce.hadoop.MRJobConfig.MRR_INTERMEDIATE_STAGES,
- iReduceStagesCount);
- LOG.info("Running MRR with " + iReduceStagesCount + " IR stages");
-
- for (int i = 1; i <= iReduceStagesCount; ++i) {
- // Set reducer class for intermediate reduce
- conf.setClass(
- MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,
- "mapreduce.job.reduce.class"), ISleepReducer.class, Reducer.class);
- // Set reducer output key class
- conf.setClass(
- MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,
- "mapreduce.map.output.key.class"), IntWritable.class, Object.class);
- // Set reducer output value class
- conf.setClass(
- MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,
- "mapreduce.map.output.value.class"), IntWritable.class, Object.class);
- conf.setInt(
- MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,
- "mapreduce.job.reduces"), numIReducer);
- }
-
- Job job = Job.getInstance(conf, "sleep");
- job.setNumReduceTasks(numReducer);
- job.setJarByClass(MRRSleepJob.class);
- job.setNumReduceTasks(numReducer);
- job.setMapperClass(SleepMapper.class);
- job.setMapOutputKeyClass(IntWritable.class);
- job.setMapOutputValueClass(IntWritable.class);
- job.setReducerClass(SleepReducer.class);
- job.setOutputFormatClass(NullOutputFormat.class);
- job.setInputFormatClass(SleepInputFormat.class);
- job.setPartitionerClass(MRRSleepJobPartitioner.class);
- job.setSpeculativeExecution(false);
- job.setJobName("Sleep job");
-
- FileInputFormat.addInputPath(job, new Path("ignored"));
- return job;
- }
-
- public int run(String[] args) throws Exception {
-
- if(args.length < 1) {
- System.err.println("MRRSleepJob [-m numMapper] [-r numReducer]" +
- " [-ir numIntermediateReducer]" +
- " [-irs numIntermediateReducerStages]" +
- " [-mt mapSleepTime (msec)] [-rt reduceSleepTime (msec)]" +
- " [-irt intermediateReduceSleepTime]" +
- " [-recordt recordSleepTime (msec)]" +
- " [-generateSplitsInAM (false)/true]" +
- " [-writeSplitsToDfs (false)/true]");
- ToolRunner.printGenericCommandUsage(System.err);
- return 2;
- }
-
- int numMapper = 1, numReducer = 1, numIReducer = 1;
- long mapSleepTime = 100, reduceSleepTime = 100, recSleepTime = 100,
- iReduceSleepTime=1;
- int mapSleepCount = 1, reduceSleepCount = 1, iReduceSleepCount = 1;
- int iReduceStagesCount = 1;
- boolean writeSplitsToDfs = false;
- boolean generateSplitsInAM = false;
- boolean splitsOptionFound = false;
-
- for(int i=0; i < args.length; i++ ) {
- if(args[i].equals("-m")) {
- numMapper = Integer.parseInt(args[++i]);
- }
- else if(args[i].equals("-r")) {
- numReducer = Integer.parseInt(args[++i]);
- }
- else if(args[i].equals("-ir")) {
- numIReducer = Integer.parseInt(args[++i]);
- }
- else if(args[i].equals("-mt")) {
- mapSleepTime = Long.parseLong(args[++i]);
- }
- else if(args[i].equals("-rt")) {
- reduceSleepTime = Long.parseLong(args[++i]);
- }
- else if(args[i].equals("-irt")) {
- iReduceSleepTime = Long.parseLong(args[++i]);
- }
- else if(args[i].equals("-irs")) {
- iReduceStagesCount = Integer.parseInt(args[++i]);
- }
- else if (args[i].equals("-recordt")) {
- recSleepTime = Long.parseLong(args[++i]);
- }
- else if (args[i].equals("-generateSplitsInAM")) {
- if (splitsOptionFound) {
- throw new RuntimeException("Cannot use both -generateSplitsInAm and -writeSplitsToDfs together");
- }
- splitsOptionFound = true;
- generateSplitsInAM = Boolean.parseBoolean(args[++i]);
-
- }
- else if (args[i].equals("-writeSplitsToDfs")) {
- if (splitsOptionFound) {
- throw new RuntimeException("Cannot use both -generateSplitsInAm and -writeSplitsToDfs together");
- }
- splitsOptionFound = true;
- writeSplitsToDfs = Boolean.parseBoolean(args[++i]);
- }
- }
-
- if (numIReducer > 0 && numReducer <= 0) {
- throw new RuntimeException("Cannot have intermediate reduces without"
- + " a final reduce");
- }
-
- // sleep for *SleepTime duration in Task by recSleepTime per record
- mapSleepCount = (int)Math.ceil(mapSleepTime / ((double)recSleepTime));
- reduceSleepCount = (int)Math.ceil(reduceSleepTime / ((double)recSleepTime));
- iReduceSleepCount = (int)Math.ceil(iReduceSleepTime / ((double)recSleepTime));
-
- TezConfiguration conf = new TezConfiguration(getConf());
- FileSystem remoteFs = FileSystem.get(conf);
-
- conf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
- conf.get(
- TezConfiguration.TEZ_AM_STAGING_DIR,
- TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT));
-
- Path remoteStagingDir =
- remoteFs.makeQualified(new Path(conf.get(
- TezConfiguration.TEZ_AM_STAGING_DIR,
- TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT),
- Long.toString(System.currentTimeMillis())));
- TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);
-
- DAG dag = createDAG(remoteFs, conf, remoteStagingDir,
- numMapper, numReducer, iReduceStagesCount, numIReducer,
- mapSleepTime, mapSleepCount, reduceSleepTime, reduceSleepCount,
- iReduceSleepTime, iReduceSleepCount, writeSplitsToDfs, generateSplitsInAM);
-
- TezClient tezSession = new TezClient("MRRSleep", conf, false, null, credentials);
- tezSession.start();
- DAGClient dagClient = tezSession.submitDAG(dag);
-
- while (true) {
- DAGStatus status = dagClient.getDAGStatus(null);
- LOG.info("DAG Status: " + status);
- if (status.isCompleted()) {
- break;
- }
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- // do nothing
- }
- }
- tezSession.stop();
-
- return dagClient.getDAGStatus(null).getState().equals(DAGStatus.State.SUCCEEDED) ? 0 : 1;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MapredWordCount.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MapredWordCount.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MapredWordCount.java
deleted file mode 100644
index 6e7300c..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MapredWordCount.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.StringTokenizer;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MapReduceBase;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.tez.examples.WordCount;
-
-/**
- * This is an example Hadoop Map/Reduce application using the mapred apis.
- * It reads the text input files, breaks each line into words
- * and counts them. The output is a locally sorted list of words and the
- * count of how often they occurred.
- *
- * To run: bin/hadoop jar examples.jar wordcount
- * [-m <i>maps</i>] [-r <i>reduces</i>] <i>in-dir</i> <i>out-dir</i>
- */
-public class MapredWordCount extends Configured implements Tool {
-
- private static final Log LOG = LogFactory.getLog(MapredWordCount.class);
-
- /**
- * Counts the words in each line.
- * For each line of input, break the line into words and emit them as
- * (<b>word</b>, <b>1</b>).
- */
- public static class MapClass extends MapReduceBase
- implements Mapper<LongWritable, Text, Text, IntWritable> {
-
- private final static IntWritable one = new IntWritable(1);
- private Text word = new Text();
-
- public void map(LongWritable key, Text value,
- OutputCollector<Text, IntWritable> output,
- Reporter reporter) throws IOException {
- String line = value.toString();
- StringTokenizer itr = new StringTokenizer(line);
- while (itr.hasMoreTokens()) {
- word.set(itr.nextToken());
- output.collect(word, one);
- }
- }
- }
-
- /**
- * A reducer class that just emits the sum of the input values.
- */
- public static class Reduce extends MapReduceBase
- implements Reducer<Text, IntWritable, Text, IntWritable> {
-
- public void reduce(Text key, Iterator<IntWritable> values,
- OutputCollector<Text, IntWritable> output,
- Reporter reporter) throws IOException {
- int sum = 0;
- while (values.hasNext()) {
- sum += values.next().get();
- }
- output.collect(key, new IntWritable(sum));
- }
- }
-
- static int printUsage() {
- System.out.println("wordcount [-m <maps>] [-r <reduces>] <input> <output>");
- ToolRunner.printGenericCommandUsage(System.out);
- return -1;
- }
-
- /**
- * The main driver for word count map/reduce program.
- * Invoke this method to submit the map/reduce job.
- * @throws IOException When there is communication problems with the
- * job tracker.
- */
- public int run(String[] args) throws Exception {
- JobConf conf = new JobConf(getConf(), WordCount.class);
- conf.setJobName("wordcount");
- LOG.info("Running WordCount job using mapred apis");
-
- // the keys are words (strings)
- conf.setOutputKeyClass(Text.class);
- // the values are counts (ints)
- conf.setOutputValueClass(IntWritable.class);
-
- conf.setMapperClass(MapClass.class);
- conf.setCombinerClass(Reduce.class);
- conf.setReducerClass(Reduce.class);
-
- List<String> other_args = new ArrayList<String>();
- for(int i=0; i < args.length; ++i) {
- try {
- if ("-m".equals(args[i])) {
- conf.setNumMapTasks(Integer.parseInt(args[++i]));
- } else if ("-r".equals(args[i])) {
- conf.setNumReduceTasks(Integer.parseInt(args[++i]));
- } else {
- other_args.add(args[i]);
- }
- } catch (NumberFormatException except) {
- LOG.error("Integer expected instead of " + args[i]);
- return printUsage();
- } catch (ArrayIndexOutOfBoundsException except) {
- LOG.error("Required parameter missing from " + args[i-1]);
- return printUsage();
- }
- }
- // Make sure there are exactly 2 parameters left.
- if (other_args.size() != 2) {
- LOG.error("Wrong number of parameters: " +
- other_args.size() + " instead of 2.");
- return printUsage();
- }
- FileInputFormat.setInputPaths(conf, other_args.get(0));
- FileOutputFormat.setOutputPath(conf, new Path(other_args.get(1)));
-
- JobClient.runJob(conf);
- return 0;
- }
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(),
- new MapredWordCount(), args);
- System.exit(res);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java
deleted file mode 100644
index 8251b78..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java
+++ /dev/null
@@ -1,757 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.ClusterStatus;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-/**
- * This program uses map/reduce to just run a distributed job where there is
- * no interaction between the tasks and each task writes a large unsorted
- * random sequence of words.
- * In order for this program to generate data for terasort with a 5-10 words
- * per key and 20-100 words per value, have the following config:
- * <xmp>
- * <?xml version="1.0"?>
- * <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
- * <configuration>
- * <property>
- * <name>mapreduce.randomtextwriter.minwordskey</name>
- * <value>5</value>
- * </property>
- * <property>
- * <name>mapreduce.randomtextwriter.maxwordskey</name>
- * <value>10</value>
- * </property>
- * <property>
- * <name>mapreduce.randomtextwriter.minwordsvalue</name>
- * <value>20</value>
- * </property>
- * <property>
- * <name>mapreduce.randomtextwriter.maxwordsvalue</name>
- * <value>100</value>
- * </property>
- * <property>
- * <name>mapreduce.randomtextwriter.totalbytes</name>
- * <value>1099511627776</value>
- * </property>
- * </configuration></xmp>
- *
- * Equivalently, {@link RandomTextWriter} also supports all the above options
- * and ones supported by {@link Tool} via the command-line.
- *
- * To run: bin/hadoop jar hadoop-${version}-examples.jar randomtextwriter
- * [-outFormat <i>output format class</i>] <i>output</i>
- */
-public class RandomTextWriter extends Configured implements Tool {
- public static final String TOTAL_BYTES =
- "mapreduce.randomtextwriter.totalbytes";
- public static final String BYTES_PER_MAP =
- "mapreduce.randomtextwriter.bytespermap";
- public static final String MAPS_PER_HOST =
- "mapreduce.randomtextwriter.mapsperhost";
- public static final String MAX_VALUE = "mapreduce.randomtextwriter.maxwordsvalue";
- public static final String MIN_VALUE = "mapreduce.randomtextwriter.minwordsvalue";
- public static final String MIN_KEY = "mapreduce.randomtextwriter.minwordskey";
- public static final String MAX_KEY = "mapreduce.randomtextwriter.maxwordskey";
-
- static int printUsage() {
- System.out.println("randomtextwriter " +
- "[-outFormat <output format class>] " +
- "<output>");
- ToolRunner.printGenericCommandUsage(System.out);
- return 2;
- }
-
- /**
- * User counters
- */
- static enum Counters { RECORDS_WRITTEN, BYTES_WRITTEN }
-
- static class RandomTextMapper extends Mapper<Text, Text, Text, Text> {
-
- private long numBytesToWrite;
- private int minWordsInKey;
- private int wordsInKeyRange;
- private int minWordsInValue;
- private int wordsInValueRange;
- private Random random = new Random();
-
- /**
- * Save the configuration value that we need to write the data.
- */
- public void setup(Context context) {
- Configuration conf = context.getConfiguration();
- numBytesToWrite = conf.getLong(BYTES_PER_MAP,
- 1*1024*1024*1024);
- minWordsInKey = conf.getInt(MIN_KEY, 5);
- wordsInKeyRange = (conf.getInt(MAX_KEY, 10) - minWordsInKey);
- minWordsInValue = conf.getInt(MIN_VALUE, 10);
- wordsInValueRange = (conf.getInt(MAX_VALUE, 100) - minWordsInValue);
- }
-
- /**
- * Given an output filename, write a bunch of random records to it.
- */
- public void map(Text key, Text value,
- Context context) throws IOException,InterruptedException {
- int itemCount = 0;
- while (numBytesToWrite > 0) {
- // Generate the key/value
- int noWordsKey = minWordsInKey +
- (wordsInKeyRange != 0 ? random.nextInt(wordsInKeyRange) : 0);
- int noWordsValue = minWordsInValue +
- (wordsInValueRange != 0 ? random.nextInt(wordsInValueRange) : 0);
- Text keyWords = generateSentence(noWordsKey);
- Text valueWords = generateSentence(noWordsValue);
-
- // Write the sentence
- context.write(keyWords, valueWords);
-
- numBytesToWrite -= (keyWords.getLength() + valueWords.getLength());
-
- // Update counters, progress etc.
- context.getCounter(Counters.BYTES_WRITTEN).increment(
- keyWords.getLength() + valueWords.getLength());
- context.getCounter(Counters.RECORDS_WRITTEN).increment(1);
- if (++itemCount % 200 == 0) {
- context.setStatus("wrote record " + itemCount + ". " +
- numBytesToWrite + " bytes left.");
- }
- }
- context.setStatus("done with " + itemCount + " records.");
- }
-
- private Text generateSentence(int noWords) {
- StringBuffer sentence = new StringBuffer();
- String space = " ";
- for (int i=0; i < noWords; ++i) {
- sentence.append(words[random.nextInt(words.length)]);
- sentence.append(space);
- }
- return new Text(sentence.toString());
- }
- }
-
- /**
- * This is the main routine for launching a distributed random write job.
- * It runs 10 maps/node and each node writes 1 gig of data to a DFS file.
- * The reduce doesn't do anything.
- *
- * @throws IOException
- */
- public int run(String[] args) throws Exception {
- if (args.length == 0) {
- return printUsage();
- }
-
- Configuration conf = getConf();
- JobClient client = new JobClient(conf);
- ClusterStatus cluster = client.getClusterStatus();
- int numMapsPerHost = conf.getInt(MAPS_PER_HOST, 10);
- long numBytesToWritePerMap = conf.getLong(BYTES_PER_MAP,
- 1*1024*1024*1024);
- if (numBytesToWritePerMap == 0) {
- System.err.println("Cannot have " + BYTES_PER_MAP +" set to 0");
- return -2;
- }
- long totalBytesToWrite = conf.getLong(TOTAL_BYTES,
- numMapsPerHost*numBytesToWritePerMap*cluster.getTaskTrackers());
- int numMaps = (int) (totalBytesToWrite / numBytesToWritePerMap);
- if (numMaps == 0 && totalBytesToWrite > 0) {
- numMaps = 1;
- conf.setLong(BYTES_PER_MAP, totalBytesToWrite);
- }
- conf.setInt(MRJobConfig.NUM_MAPS, numMaps);
-
- Job job = new Job(conf);
-
- job.setJarByClass(RandomTextWriter.class);
- job.setJobName("random-text-writer");
-
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(Text.class);
-
- job.setInputFormatClass(RandomWriter.RandomInputFormat.class);
- job.setMapperClass(RandomTextMapper.class);
-
- Class<? extends OutputFormat> outputFormatClass =
- SequenceFileOutputFormat.class;
- List<String> otherArgs = new ArrayList<String>();
- for(int i=0; i < args.length; ++i) {
- try {
- if ("-outFormat".equals(args[i])) {
- outputFormatClass =
- Class.forName(args[++i]).asSubclass(OutputFormat.class);
- } else {
- otherArgs.add(args[i]);
- }
- } catch (ArrayIndexOutOfBoundsException except) {
- System.out.println("ERROR: Required parameter missing from " +
- args[i-1]);
- return printUsage(); // exits
- }
- }
-
- job.setOutputFormatClass(outputFormatClass);
- FileOutputFormat.setOutputPath(job, new Path(otherArgs.get(0)));
-
- System.out.println("Running " + numMaps + " maps.");
-
- // reducer NONE
- job.setNumReduceTasks(0);
-
- Date startTime = new Date();
- System.out.println("Job started: " + startTime);
- int ret = job.waitForCompletion(true) ? 0 : 1;
- Date endTime = new Date();
- System.out.println("Job ended: " + endTime);
- System.out.println("The job took " +
- (endTime.getTime() - startTime.getTime()) /1000 +
- " seconds.");
-
- return ret;
- }
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new RandomTextWriter(), args);
- System.exit(res);
- }
-
- /**
- * A random list of 100 words from /usr/share/dict/words
- */
- private static String[] words = {
- "diurnalness", "Homoiousian",
- "spiranthic", "tetragynian",
- "silverhead", "ungreat",
- "lithograph", "exploiter",
- "physiologian", "by",
- "hellbender", "Filipendula",
- "undeterring", "antiscolic",
- "pentagamist", "hypoid",
- "cacuminal", "sertularian",
- "schoolmasterism", "nonuple",
- "gallybeggar", "phytonic",
- "swearingly", "nebular",
- "Confervales", "thermochemically",
- "characinoid", "cocksuredom",
- "fallacious", "feasibleness",
- "debromination", "playfellowship",
- "tramplike", "testa",
- "participatingly", "unaccessible",
- "bromate", "experientialist",
- "roughcast", "docimastical",
- "choralcelo", "blightbird",
- "peptonate", "sombreroed",
- "unschematized", "antiabolitionist",
- "besagne", "mastication",
- "bromic", "sviatonosite",
- "cattimandoo", "metaphrastical",
- "endotheliomyoma", "hysterolysis",
- "unfulminated", "Hester",
- "oblongly", "blurredness",
- "authorling", "chasmy",
- "Scorpaenidae", "toxihaemia",
- "Dictograph", "Quakerishly",
- "deaf", "timbermonger",
- "strammel", "Thraupidae",
- "seditious", "plerome",
- "Arneb", "eristically",
- "serpentinic", "glaumrie",
- "socioromantic", "apocalypst",
- "tartrous", "Bassaris",
- "angiolymphoma", "horsefly",
- "kenno", "astronomize",
- "euphemious", "arsenide",
- "untongued", "parabolicness",
- "uvanite", "helpless",
- "gemmeous", "stormy",
- "templar", "erythrodextrin",
- "comism", "interfraternal",
- "preparative", "parastas",
- "frontoorbital", "Ophiosaurus",
- "diopside", "serosanguineous",
- "ununiformly", "karyological",
- "collegian", "allotropic",
- "depravity", "amylogenesis",
- "reformatory", "epidymides",
- "pleurotropous", "trillium",
- "dastardliness", "coadvice",
- "embryotic", "benthonic",
- "pomiferous", "figureheadship",
- "Megaluridae", "Harpa",
- "frenal", "commotion",
- "abthainry", "cobeliever",
- "manilla", "spiciferous",
- "nativeness", "obispo",
- "monilioid", "biopsic",
- "valvula", "enterostomy",
- "planosubulate", "pterostigma",
- "lifter", "triradiated",
- "venialness", "tum",
- "archistome", "tautness",
- "unswanlike", "antivenin",
- "Lentibulariaceae", "Triphora",
- "angiopathy", "anta",
- "Dawsonia", "becomma",
- "Yannigan", "winterproof",
- "antalgol", "harr",
- "underogating", "ineunt",
- "cornberry", "flippantness",
- "scyphostoma", "approbation",
- "Ghent", "Macraucheniidae",
- "scabbiness", "unanatomized",
- "photoelasticity", "eurythermal",
- "enation", "prepavement",
- "flushgate", "subsequentially",
- "Edo", "antihero",
- "Isokontae", "unforkedness",
- "porriginous", "daytime",
- "nonexecutive", "trisilicic",
- "morphiomania", "paranephros",
- "botchedly", "impugnation",
- "Dodecatheon", "obolus",
- "unburnt", "provedore",
- "Aktistetae", "superindifference",
- "Alethea", "Joachimite",
- "cyanophilous", "chorograph",
- "brooky", "figured",
- "periclitation", "quintette",
- "hondo", "ornithodelphous",
- "unefficient", "pondside",
- "bogydom", "laurinoxylon",
- "Shiah", "unharmed",
- "cartful", "noncrystallized",
- "abusiveness", "cromlech",
- "japanned", "rizzomed",
- "underskin", "adscendent",
- "allectory", "gelatinousness",
- "volcano", "uncompromisingly",
- "cubit", "idiotize",
- "unfurbelowed", "undinted",
- "magnetooptics", "Savitar",
- "diwata", "ramosopalmate",
- "Pishquow", "tomorn",
- "apopenptic", "Haversian",
- "Hysterocarpus", "ten",
- "outhue", "Bertat",
- "mechanist", "asparaginic",
- "velaric", "tonsure",
- "bubble", "Pyrales",
- "regardful", "glyphography",
- "calabazilla", "shellworker",
- "stradametrical", "havoc",
- "theologicopolitical", "sawdust",
- "diatomaceous", "jajman",
- "temporomastoid", "Serrifera",
- "Ochnaceae", "aspersor",
- "trailmaking", "Bishareen",
- "digitule", "octogynous",
- "epididymitis", "smokefarthings",
- "bacillite", "overcrown",
- "mangonism", "sirrah",
- "undecorated", "psychofugal",
- "bismuthiferous", "rechar",
- "Lemuridae", "frameable",
- "thiodiazole", "Scanic",
- "sportswomanship", "interruptedness",
- "admissory", "osteopaedion",
- "tingly", "tomorrowness",
- "ethnocracy", "trabecular",
- "vitally", "fossilism",
- "adz", "metopon",
- "prefatorial", "expiscate",
- "diathermacy", "chronist",
- "nigh", "generalizable",
- "hysterogen", "aurothiosulphuric",
- "whitlowwort", "downthrust",
- "Protestantize", "monander",
- "Itea", "chronographic",
- "silicize", "Dunlop",
- "eer", "componental",
- "spot", "pamphlet",
- "antineuritic", "paradisean",
- "interruptor", "debellator",
- "overcultured", "Florissant",
- "hyocholic", "pneumatotherapy",
- "tailoress", "rave",
- "unpeople", "Sebastian",
- "thermanesthesia", "Coniferae",
- "swacking", "posterishness",
- "ethmopalatal", "whittle",
- "analgize", "scabbardless",
- "naught", "symbiogenetically",
- "trip", "parodist",
- "columniform", "trunnel",
- "yawler", "goodwill",
- "pseudohalogen", "swangy",
- "cervisial", "mediateness",
- "genii", "imprescribable",
- "pony", "consumptional",
- "carposporangial", "poleax",
- "bestill", "subfebrile",
- "sapphiric", "arrowworm",
- "qualminess", "ultraobscure",
- "thorite", "Fouquieria",
- "Bermudian", "prescriber",
- "elemicin", "warlike",
- "semiangle", "rotular",
- "misthread", "returnability",
- "seraphism", "precostal",
- "quarried", "Babylonism",
- "sangaree", "seelful",
- "placatory", "pachydermous",
- "bozal", "galbulus",
- "spermaphyte", "cumbrousness",
- "pope", "signifier",
- "Endomycetaceae", "shallowish",
- "sequacity", "periarthritis",
- "bathysphere", "pentosuria",
- "Dadaism", "spookdom",
- "Consolamentum", "afterpressure",
- "mutter", "louse",
- "ovoviviparous", "corbel",
- "metastoma", "biventer",
- "Hydrangea", "hogmace",
- "seizing", "nonsuppressed",
- "oratorize", "uncarefully",
- "benzothiofuran", "penult",
- "balanocele", "macropterous",
- "dishpan", "marten",
- "absvolt", "jirble",
- "parmelioid", "airfreighter",
- "acocotl", "archesporial",
- "hypoplastral", "preoral",
- "quailberry", "cinque",
- "terrestrially", "stroking",
- "limpet", "moodishness",
- "canicule", "archididascalian",
- "pompiloid", "overstaid",
- "introducer", "Italical",
- "Christianopaganism", "prescriptible",
- "subofficer", "danseuse",
- "cloy", "saguran",
- "frictionlessly", "deindividualization",
- "Bulanda", "ventricous",
- "subfoliar", "basto",
- "scapuloradial", "suspend",
- "stiffish", "Sphenodontidae",
- "eternal", "verbid",
- "mammonish", "upcushion",
- "barkometer", "concretion",
- "preagitate", "incomprehensible",
- "tristich", "visceral",
- "hemimelus", "patroller",
- "stentorophonic", "pinulus",
- "kerykeion", "brutism",
- "monstership", "merciful",
- "overinstruct", "defensibly",
- "bettermost", "splenauxe",
- "Mormyrus", "unreprimanded",
- "taver", "ell",
- "proacquittal", "infestation",
- "overwoven", "Lincolnlike",
- "chacona", "Tamil",
- "classificational", "lebensraum",
- "reeveland", "intuition",
- "Whilkut", "focaloid",
- "Eleusinian", "micromembrane",
- "byroad", "nonrepetition",
- "bacterioblast", "brag",
- "ribaldrous", "phytoma",
- "counteralliance", "pelvimetry",
- "pelf", "relaster",
- "thermoresistant", "aneurism",
- "molossic", "euphonym",
- "upswell", "ladhood",
- "phallaceous", "inertly",
- "gunshop", "stereotypography",
- "laryngic", "refasten",
- "twinling", "oflete",
- "hepatorrhaphy", "electrotechnics",
- "cockal", "guitarist",
- "topsail", "Cimmerianism",
- "larklike", "Llandovery",
- "pyrocatechol", "immatchable",
- "chooser", "metrocratic",
- "craglike", "quadrennial",
- "nonpoisonous", "undercolored",
- "knob", "ultratense",
- "balladmonger", "slait",
- "sialadenitis", "bucketer",
- "magnificently", "unstipulated",
- "unscourged", "unsupercilious",
- "packsack", "pansophism",
- "soorkee", "percent",
- "subirrigate", "champer",
- "metapolitics", "spherulitic",
- "involatile", "metaphonical",
- "stachyuraceous", "speckedness",
- "bespin", "proboscidiform",
- "gul", "squit",
- "yeelaman", "peristeropode",
- "opacousness", "shibuichi",
- "retinize", "yote",
- "misexposition", "devilwise",
- "pumpkinification", "vinny",
- "bonze", "glossing",
- "decardinalize", "transcortical",
- "serphoid", "deepmost",
- "guanajuatite", "wemless",
- "arval", "lammy",
- "Effie", "Saponaria",
- "tetrahedral", "prolificy",
- "excerpt", "dunkadoo",
- "Spencerism", "insatiately",
- "Gilaki", "oratorship",
- "arduousness", "unbashfulness",
- "Pithecolobium", "unisexuality",
- "veterinarian", "detractive",
- "liquidity", "acidophile",
- "proauction", "sural",
- "totaquina", "Vichyite",
- "uninhabitedness", "allegedly",
- "Gothish", "manny",
- "Inger", "flutist",
- "ticktick", "Ludgatian",
- "homotransplant", "orthopedical",
- "diminutively", "monogoneutic",
- "Kenipsim", "sarcologist",
- "drome", "stronghearted",
- "Fameuse", "Swaziland",
- "alen", "chilblain",
- "beatable", "agglomeratic",
- "constitutor", "tendomucoid",
- "porencephalous", "arteriasis",
- "boser", "tantivy",
- "rede", "lineamental",
- "uncontradictableness", "homeotypical",
- "masa", "folious",
- "dosseret", "neurodegenerative",
- "subtransverse", "Chiasmodontidae",
- "palaeotheriodont", "unstressedly",
- "chalcites", "piquantness",
- "lampyrine", "Aplacentalia",
- "projecting", "elastivity",
- "isopelletierin", "bladderwort",
- "strander", "almud",
- "iniquitously", "theologal",
- "bugre", "chargeably",
- "imperceptivity", "meriquinoidal",
- "mesophyte", "divinator",
- "perfunctory", "counterappellant",
- "synovial", "charioteer",
- "crystallographical", "comprovincial",
- "infrastapedial", "pleasurehood",
- "inventurous", "ultrasystematic",
- "subangulated", "supraoesophageal",
- "Vaishnavism", "transude",
- "chrysochrous", "ungrave",
- "reconciliable", "uninterpleaded",
- "erlking", "wherefrom",
- "aprosopia", "antiadiaphorist",
- "metoxazine", "incalculable",
- "umbellic", "predebit",
- "foursquare", "unimmortal",
- "nonmanufacture", "slangy",
- "predisputant", "familist",
- "preaffiliate", "friarhood",
- "corelysis", "zoonitic",
- "halloo", "paunchy",
- "neuromimesis", "aconitine",
- "hackneyed", "unfeeble",
- "cubby", "autoschediastical",
- "naprapath", "lyrebird",
- "inexistency", "leucophoenicite",
- "ferrogoslarite", "reperuse",
- "uncombable", "tambo",
- "propodiale", "diplomatize",
- "Russifier", "clanned",
- "corona", "michigan",
- "nonutilitarian", "transcorporeal",
- "bought", "Cercosporella",
- "stapedius", "glandularly",
- "pictorially", "weism",
- "disilane", "rainproof",
- "Caphtor", "scrubbed",
- "oinomancy", "pseudoxanthine",
- "nonlustrous", "redesertion",
- "Oryzorictinae", "gala",
- "Mycogone", "reappreciate",
- "cyanoguanidine", "seeingness",
- "breadwinner", "noreast",
- "furacious", "epauliere",
- "omniscribent", "Passiflorales",
- "uninductive", "inductivity",
- "Orbitolina", "Semecarpus",
- "migrainoid", "steprelationship",
- "phlogisticate", "mesymnion",
- "sloped", "edificator",
- "beneficent", "culm",
- "paleornithology", "unurban",
- "throbless", "amplexifoliate",
- "sesquiquintile", "sapience",
- "astucious", "dithery",
- "boor", "ambitus",
- "scotching", "uloid",
- "uncompromisingness", "hoove",
- "waird", "marshiness",
- "Jerusalem", "mericarp",
- "unevoked", "benzoperoxide",
- "outguess", "pyxie",
- "hymnic", "euphemize",
- "mendacity", "erythremia",
- "rosaniline", "unchatteled",
- "lienteria", "Bushongo",
- "dialoguer", "unrepealably",
- "rivethead", "antideflation",
- "vinegarish", "manganosiderite",
- "doubtingness", "ovopyriform",
- "Cephalodiscus", "Muscicapa",
- "Animalivora", "angina",
- "planispheric", "ipomoein",
- "cuproiodargyrite", "sandbox",
- "scrat", "Munnopsidae",
- "shola", "pentafid",
- "overstudiousness", "times",
- "nonprofession", "appetible",
- "valvulotomy", "goladar",
- "uniarticular", "oxyterpene",
- "unlapsing", "omega",
- "trophonema", "seminonflammable",
- "circumzenithal", "starer",
- "depthwise", "liberatress",
- "unleavened", "unrevolting",
- "groundneedle", "topline",
- "wandoo", "umangite",
- "ordinant", "unachievable",
- "oversand", "snare",
- "avengeful", "unexplicit",
- "mustafina", "sonable",
- "rehabilitative", "eulogization",
- "papery", "technopsychology",
- "impressor", "cresylite",
- "entame", "transudatory",
- "scotale", "pachydermatoid",
- "imaginary", "yeat",
- "slipped", "stewardship",
- "adatom", "cockstone",
- "skyshine", "heavenful",
- "comparability", "exprobratory",
- "dermorhynchous", "parquet",
- "cretaceous", "vesperal",
- "raphis", "undangered",
- "Glecoma", "engrain",
- "counteractively", "Zuludom",
- "orchiocatabasis", "Auriculariales",
- "warriorwise", "extraorganismal",
- "overbuilt", "alveolite",
- "tetchy", "terrificness",
- "widdle", "unpremonished",
- "rebilling", "sequestrum",
- "equiconvex", "heliocentricism",
- "catabaptist", "okonite",
- "propheticism", "helminthagogic",
- "calycular", "giantly",
- "wingable", "golem",
- "unprovided", "commandingness",
- "greave", "haply",
- "doina", "depressingly",
- "subdentate", "impairment",
- "decidable", "neurotrophic",
- "unpredict", "bicorporeal",
- "pendulant", "flatman",
- "intrabred", "toplike",
- "Prosobranchiata", "farrantly",
- "toxoplasmosis", "gorilloid",
- "dipsomaniacal", "aquiline",
- "atlantite", "ascitic",
- "perculsive", "prospectiveness",
- "saponaceous", "centrifugalization",
- "dinical", "infravaginal",
- "beadroll", "affaite",
- "Helvidian", "tickleproof",
- "abstractionism", "enhedge",
- "outwealth", "overcontribute",
- "coldfinch", "gymnastic",
- "Pincian", "Munychian",
- "codisjunct", "quad",
- "coracomandibular", "phoenicochroite",
- "amender", "selectivity",
- "putative", "semantician",
- "lophotrichic", "Spatangoidea",
- "saccharogenic", "inferent",
- "Triconodonta", "arrendation",
- "sheepskin", "taurocolla",
- "bunghole", "Machiavel",
- "triakistetrahedral", "dehairer",
- "prezygapophysial", "cylindric",
- "pneumonalgia", "sleigher",
- "emir", "Socraticism",
- "licitness", "massedly",
- "instructiveness", "sturdied",
- "redecrease", "starosta",
- "evictor", "orgiastic",
- "squdge", "meloplasty",
- "Tsonecan", "repealableness",
- "swoony", "myesthesia",
- "molecule", "autobiographist",
- "reciprocation", "refective",
- "unobservantness", "tricae",
- "ungouged", "floatability",
- "Mesua", "fetlocked",
- "chordacentrum", "sedentariness",
- "various", "laubanite",
- "nectopod", "zenick",
- "sequentially", "analgic",
- "biodynamics", "posttraumatic",
- "nummi", "pyroacetic",
- "bot", "redescend",
- "dispermy", "undiffusive",
- "circular", "trillion",
- "Uraniidae", "ploration",
- "discipular", "potentness",
- "sud", "Hu",
- "Eryon", "plugger",
- "subdrainage", "jharal",
- "abscission", "supermarket",
- "countergabion", "glacierist",
- "lithotresis", "minniebush",
- "zanyism", "eucalypteol",
- "sterilely", "unrealize",
- "unpatched", "hypochondriacism",
- "critically", "cheesecutter",
- };
-}
[05/10] TEZ-1055. Rename tez-mapreduce-examples to tez-examples
(Hitesh Shah via bikas)
Posted by bi...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraValidate.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraValidate.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraValidate.java
deleted file mode 100644
index 24c3145..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraValidate.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples.terasort;
-
-import java.io.IOException;
-import java.util.zip.Checksum;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.util.PureJavaCrc32;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-/**
- * Generate 1 mapper per a file that checks to make sure the keys
- * are sorted within each file. The mapper also generates
- * "$file:begin", first key and "$file:end", last key. The reduce verifies that
- * all of the start/end items are in order.
- * Any output from the reduce is problem report.
- * <p>
- * To run the program:
- * <b>bin/hadoop jar hadoop-*-examples.jar teravalidate out-dir report-dir</b>
- * <p>
- * If there is any output, something is wrong and the output of the reduce
- * will have the problem report.
- */
-public class TeraValidate extends Configured implements Tool {
- private static final Text ERROR = new Text("error");
- private static final Text CHECKSUM = new Text("checksum");
-
- private static String textifyBytes(Text t) {
- BytesWritable b = new BytesWritable();
- b.set(t.getBytes(), 0, t.getLength());
- return b.toString();
- }
-
- static class ValidateMapper extends Mapper<Text,Text,Text,Text> {
- private Text lastKey;
- private String filename;
- private Unsigned16 checksum = new Unsigned16();
- private Unsigned16 tmp = new Unsigned16();
- private Checksum crc32 = new PureJavaCrc32();
-
- /**
- * Get the final part of the input name
- * @param split the input split
- * @return the "part-r-00000" for the input
- */
- private String getFilename(FileSplit split) {
- return split.getPath().getName();
- }
-
- public void map(Text key, Text value, Context context)
- throws IOException, InterruptedException {
- if (lastKey == null) {
- FileSplit fs = (FileSplit) context.getInputSplit();
- filename = getFilename(fs);
- context.write(new Text(filename + ":begin"), key);
- lastKey = new Text();
- } else {
- if (key.compareTo(lastKey) < 0) {
- context.write(ERROR, new Text("misorder in " + filename +
- " between " + textifyBytes(lastKey) +
- " and " + textifyBytes(key)));
- }
- }
- // compute the crc of the key and value and add it to the sum
- crc32.reset();
- crc32.update(key.getBytes(), 0, key.getLength());
- crc32.update(value.getBytes(), 0, value.getLength());
- tmp.set(crc32.getValue());
- checksum.add(tmp);
- lastKey.set(key);
- }
-
- public void cleanup(Context context)
- throws IOException, InterruptedException {
- if (lastKey != null) {
- context.write(new Text(filename + ":end"), lastKey);
- context.write(CHECKSUM, new Text(checksum.toString()));
- }
- }
- }
-
- /**
- * Check the boundaries between the output files by making sure that the
- * boundary keys are always increasing.
- * Also passes any error reports along intact.
- */
- static class ValidateReducer extends Reducer<Text,Text,Text,Text> {
- private boolean firstKey = true;
- private Text lastKey = new Text();
- private Text lastValue = new Text();
- public void reduce(Text key, Iterable<Text> values,
- Context context) throws IOException, InterruptedException {
- if (ERROR.equals(key)) {
- for (Text val : values) {
- context.write(key, val);
- }
- } else if (CHECKSUM.equals(key)) {
- Unsigned16 tmp = new Unsigned16();
- Unsigned16 sum = new Unsigned16();
- for (Text val : values) {
- tmp.set(val.toString());
- sum.add(tmp);
- }
- context.write(CHECKSUM, new Text(sum.toString()));
- } else {
- Text value = values.iterator().next();
- if (firstKey) {
- firstKey = false;
- } else {
- if (value.compareTo(lastValue) < 0) {
- context.write(ERROR,
- new Text("bad key partitioning:\n file " +
- lastKey + " key " +
- textifyBytes(lastValue) +
- "\n file " + key + " key " +
- textifyBytes(value)));
- }
- }
- lastKey.set(key);
- lastValue.set(value);
- }
- }
-
- }
-
- private static void usage() throws IOException {
- System.err.println("teravalidate <out-dir> <report-dir>");
- }
-
- public int run(String[] args) throws Exception {
- Job job = Job.getInstance(getConf());
- if (args.length != 2) {
- usage();
- return 1;
- }
- TeraInputFormat.setInputPaths(job, new Path(args[0]));
- FileOutputFormat.setOutputPath(job, new Path(args[1]));
- job.setJobName("TeraValidate");
- job.setJarByClass(TeraValidate.class);
- job.setMapperClass(ValidateMapper.class);
- job.setReducerClass(ValidateReducer.class);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(Text.class);
- // force a single reducer
- job.setNumReduceTasks(1);
- // force a single split
- FileInputFormat.setMinInputSplitSize(job, Long.MAX_VALUE);
- job.setInputFormatClass(TeraInputFormat.class);
- return job.waitForCompletion(true) ? 0 : 1;
- }
-
- /**
- * @param args
- */
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new TeraValidate(), args);
- System.exit(res);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/Unsigned16.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/Unsigned16.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/Unsigned16.java
deleted file mode 100644
index 1335bf4..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/Unsigned16.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples.terasort;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.Writable;
-
-/**
- * An unsigned 16 byte integer class that supports addition, multiplication,
- * and left shifts.
- */
-class Unsigned16 implements Writable {
- private long hi8;
- private long lo8;
-
- public Unsigned16() {
- hi8 = 0;
- lo8 = 0;
- }
-
- public Unsigned16(long l) {
- hi8 = 0;
- lo8 = l;
- }
-
- public Unsigned16(Unsigned16 other) {
- hi8 = other.hi8;
- lo8 = other.lo8;
- }
-
- @Override
- public boolean equals(Object o) {
- if (o instanceof Unsigned16) {
- Unsigned16 other = (Unsigned16) o;
- return other.hi8 == hi8 && other.lo8 == lo8;
- }
- return false;
- }
-
- @Override
- public int hashCode() {
- return (int) lo8;
- }
-
- /**
- * Parse a hex string
- * @param s the hex string
- */
- public Unsigned16(String s) throws NumberFormatException {
- set(s);
- }
-
- /**
- * Set the number from a hex string
- * @param s the number in hexadecimal
- * @throws NumberFormatException if the number is invalid
- */
- public void set(String s) throws NumberFormatException {
- hi8 = 0;
- lo8 = 0;
- final long lastDigit = 0xfl << 60;
- for (int i = 0; i < s.length(); ++i) {
- int digit = getHexDigit(s.charAt(i));
- if ((lastDigit & hi8) != 0) {
- throw new NumberFormatException(s + " overflowed 16 bytes");
- }
- hi8 <<= 4;
- hi8 |= (lo8 & lastDigit) >>> 60;
- lo8 <<= 4;
- lo8 |= digit;
- }
- }
-
- /**
- * Set the number to a given long.
- * @param l the new value, which is treated as an unsigned number
- */
- public void set(long l) {
- lo8 = l;
- hi8 = 0;
- }
-
- /**
- * Map a hexadecimal character into a digit.
- * @param ch the character
- * @return the digit from 0 to 15
- * @throws NumberFormatException
- */
- private static int getHexDigit(char ch) throws NumberFormatException {
- if (ch >= '0' && ch <= '9') {
- return ch - '0';
- }
- if (ch >= 'a' && ch <= 'f') {
- return ch - 'a' + 10;
- }
- if (ch >= 'A' && ch <= 'F') {
- return ch - 'A' + 10;
- }
- throw new NumberFormatException(ch + " is not a valid hex digit");
- }
-
- private static final Unsigned16 TEN = new Unsigned16(10);
-
- public static Unsigned16 fromDecimal(String s) throws NumberFormatException {
- Unsigned16 result = new Unsigned16();
- Unsigned16 tmp = new Unsigned16();
- for(int i=0; i < s.length(); i++) {
- char ch = s.charAt(i);
- if (ch < '0' || ch > '9') {
- throw new NumberFormatException(ch + " not a valid decimal digit");
- }
- int digit = ch - '0';
- result.multiply(TEN);
- tmp.set(digit);
- result.add(tmp);
- }
- return result;
- }
-
- /**
- * Return the number as a hex string.
- */
- public String toString() {
- if (hi8 == 0) {
- return Long.toHexString(lo8);
- } else {
- StringBuilder result = new StringBuilder();
- result.append(Long.toHexString(hi8));
- String loString = Long.toHexString(lo8);
- for(int i=loString.length(); i < 16; ++i) {
- result.append('0');
- }
- result.append(loString);
- return result.toString();
- }
- }
-
- /**
- * Get a given byte from the number.
- * @param b the byte to get with 0 meaning the most significant byte
- * @return the byte or 0 if b is outside of 0..15
- */
- public byte getByte(int b) {
- if (b >= 0 && b < 16) {
- if (b < 8) {
- return (byte) (hi8 >> (56 - 8*b));
- } else {
- return (byte) (lo8 >> (120 - 8*b));
- }
- }
- return 0;
- }
-
- /**
- * Get the hexadecimal digit at the given position.
- * @param p the digit position to get with 0 meaning the most significant
- * @return the character or '0' if p is outside of 0..31
- */
- public char getHexDigit(int p) {
- byte digit = getByte(p / 2);
- if (p % 2 == 0) {
- digit >>>= 4;
- }
- digit &= 0xf;
- if (digit < 10) {
- return (char) ('0' + digit);
- } else {
- return (char) ('A' + digit - 10);
- }
- }
-
- /**
- * Get the high 8 bytes as a long.
- */
- public long getHigh8() {
- return hi8;
- }
-
- /**
- * Get the low 8 bytes as a long.
- */
- public long getLow8() {
- return lo8;
- }
-
- /**
- * Multiple the current number by a 16 byte unsigned integer. Overflow is not
- * detected and the result is the low 16 bytes of the result. The numbers
- * are divided into 32 and 31 bit chunks so that the product of two chucks
- * fits in the unsigned 63 bits of a long.
- * @param b the other number
- */
- void multiply(Unsigned16 b) {
- // divide the left into 4 32 bit chunks
- long[] left = new long[4];
- left[0] = lo8 & 0xffffffffl;
- left[1] = lo8 >>> 32;
- left[2] = hi8 & 0xffffffffl;
- left[3] = hi8 >>> 32;
- // divide the right into 5 31 bit chunks
- long[] right = new long[5];
- right[0] = b.lo8 & 0x7fffffffl;
- right[1] = (b.lo8 >>> 31) & 0x7fffffffl;
- right[2] = (b.lo8 >>> 62) + ((b.hi8 & 0x1fffffffl) << 2);
- right[3] = (b.hi8 >>> 29) & 0x7fffffffl;
- right[4] = (b.hi8 >>> 60);
- // clear the cur value
- set(0);
- Unsigned16 tmp = new Unsigned16();
- for(int l=0; l < 4; ++l) {
- for (int r=0; r < 5; ++r) {
- long prod = left[l] * right[r];
- if (prod != 0) {
- int off = l*32 + r*31;
- tmp.set(prod);
- tmp.shiftLeft(off);
- add(tmp);
- }
- }
- }
- }
-
- /**
- * Add the given number into the current number.
- * @param b the other number
- */
- public void add(Unsigned16 b) {
- long sumHi;
- long sumLo;
- long reshibit, hibit0, hibit1;
-
- sumHi = hi8 + b.hi8;
-
- hibit0 = (lo8 & 0x8000000000000000L);
- hibit1 = (b.lo8 & 0x8000000000000000L);
- sumLo = lo8 + b.lo8;
- reshibit = (sumLo & 0x8000000000000000L);
- if ((hibit0 & hibit1) != 0 | ((hibit0 ^ hibit1) != 0 && reshibit == 0))
- sumHi++; /* add carry bit */
- hi8 = sumHi;
- lo8 = sumLo;
- }
-
- /**
- * Shift the number a given number of bit positions. The number is the low
- * order bits of the result.
- * @param bits the bit positions to shift by
- */
- public void shiftLeft(int bits) {
- if (bits != 0) {
- if (bits < 64) {
- hi8 <<= bits;
- hi8 |= (lo8 >>> (64 - bits));
- lo8 <<= bits;
- } else if (bits < 128) {
- hi8 = lo8 << (bits - 64);
- lo8 = 0;
- } else {
- hi8 = 0;
- lo8 = 0;
- }
- }
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- hi8 = in.readLong();
- lo8 = in.readLong();
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeLong(hi8);
- out.writeLong(lo8);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/job_history_summary.py
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/job_history_summary.py b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/job_history_summary.py
deleted file mode 100644
index 70725f8..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/job_history_summary.py
+++ /dev/null
@@ -1,100 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import re
-import sys
-
-pat = re.compile('(?P<name>[^=]+)="(?P<value>[^"]*)" *')
-counterPat = re.compile('(?P<name>[^:]+):(?P<value>[^,]*),?')
-
-def parse(tail):
- result = {}
- for n,v in re.findall(pat, tail):
- result[n] = v
- return result
-
-mapStartTime = {}
-mapEndTime = {}
-reduceStartTime = {}
-reduceShuffleTime = {}
-reduceSortTime = {}
-reduceEndTime = {}
-reduceBytes = {}
-
-for line in sys.stdin:
- words = line.split(" ",1)
- event = words[0]
- attrs = parse(words[1])
- if event == 'MapAttempt':
- if attrs.has_key("START_TIME"):
- mapStartTime[attrs["TASKID"]] = int(attrs["START_TIME"])/1000
- elif attrs.has_key("FINISH_TIME"):
- mapEndTime[attrs["TASKID"]] = int(attrs["FINISH_TIME"])/1000
- elif event == 'ReduceAttempt':
- if attrs.has_key("START_TIME"):
- reduceStartTime[attrs["TASKID"]] = int(attrs["START_TIME"]) / 1000
- elif attrs.has_key("FINISH_TIME"):
- reduceShuffleTime[attrs["TASKID"]] = int(attrs["SHUFFLE_FINISHED"])/1000
- reduceSortTime[attrs["TASKID"]] = int(attrs["SORT_FINISHED"])/1000
- reduceEndTime[attrs["TASKID"]] = int(attrs["FINISH_TIME"])/1000
- elif event == 'Task':
- if attrs["TASK_TYPE"] == "REDUCE" and attrs.has_key("COUNTERS"):
- for n,v in re.findall(counterPat, attrs["COUNTERS"]):
- if n == "File Systems.HDFS bytes written":
- reduceBytes[attrs["TASKID"]] = int(v)
-
-runningMaps = {}
-shufflingReduces = {}
-sortingReduces = {}
-runningReduces = {}
-startTime = min(reduce(min, mapStartTime.values()),
- reduce(min, reduceStartTime.values()))
-endTime = max(reduce(max, mapEndTime.values()),
- reduce(max, reduceEndTime.values()))
-
-reduces = reduceBytes.keys()
-reduces.sort()
-
-print "Name reduce-output-bytes shuffle-finish reduce-finish"
-for r in reduces:
- print r, reduceBytes[r], reduceShuffleTime[r] - startTime,
- print reduceEndTime[r] - startTime
-
-print
-
-for t in range(startTime, endTime):
- runningMaps[t] = 0
- shufflingReduces[t] = 0
- sortingReduces[t] = 0
- runningReduces[t] = 0
-
-for map in mapStartTime.keys():
- for t in range(mapStartTime[map], mapEndTime[map]):
- runningMaps[t] += 1
-for reduce in reduceStartTime.keys():
- for t in range(reduceStartTime[reduce], reduceShuffleTime[reduce]):
- shufflingReduces[t] += 1
- for t in range(reduceShuffleTime[reduce], reduceSortTime[reduce]):
- sortingReduces[t] += 1
- for t in range(reduceSortTime[reduce], reduceEndTime[reduce]):
- runningReduces[t] += 1
-
-print "time maps shuffle merge reduce"
-for t in range(startTime, endTime):
- print t - startTime, runningMaps[t], shufflingReduces[t], sortingReduces[t],
- print runningReduces[t]
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/package.html
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/package.html b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/package.html
deleted file mode 100644
index 7341c02..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/terasort/package.html
+++ /dev/null
@@ -1,114 +0,0 @@
-<html>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<body>
-This package consists of 3 map/reduce applications for Hadoop to
-compete in the annual <a
-href="http://www.hpl.hp.com/hosted/sortbenchmark" target="_top">terabyte sort</a>
-competition.
-
-<ul>
-<li><b>TeraGen</b> is a map/reduce program to generate the data.
-<li><b>TeraSort</b> samples the input data and uses map/reduce to
- sort the data into a total order.
-<li><b>TeraValidate</b> is a map/reduce program that validates the
- output is sorted.
-</ul>
-
-<p>
-
-<b>TeraGen</b> generates output data that is byte for byte
-equivalent to the C version including the newlines and specific
-keys. It divides the desired number of rows by the desired number of
-tasks and assigns ranges of rows to each map. The map jumps the random
-number generator to the correct value for the first row and generates
-the following rows.
-
-<p>
-
-<b>TeraSort</b> is a standard map/reduce sort, except for a custom
-partitioner that uses a sorted list of <i>N-1</i> sampled keys that define
-the key range for each reduce. In particular, all keys such that
-<i>sample[i-1] <= key < sample[i]</i> are sent to reduce
-<i>i</i>. This guarantees that the output of reduce <i>i</i> are all
-less than the output of reduce <i>i+1</i>. To speed up the
-partitioning, the partitioner builds a two level trie that quickly
-indexes into the list of sample keys based on the first two bytes of
-the key. TeraSort generates the sample keys by sampling the input
-before the job is submitted and writing the list of keys into HDFS.
-The input and output format, which are used by all 3 applications,
-read and write the text files in the right format. The output of the
-reduce has replication set to 1, instead of the default 3, because the
-contest does not require the output data be replicated on to multiple
-nodes.
-
-<p>
-
-<b>TeraValidate</b> ensures that the output is globally sorted. It
-creates one map per a file in the output directory and each map ensures that
-each key is less than or equal to the previous one. The map also generates
-records with the first and last keys of the file and the reduce
-ensures that the first key of file <i>i</i> is greater that the last key of
-file <i>i-1</i>. Any problems are reported as output of the reduce with the
-keys that are out of order.
-
-<p>
-
-In May 2008, Owen O'Malley ran this code on a 910 node cluster and
-sorted the 10 billion records (1 TB) in 209 seconds (3.48 minutes) to
-win the annual general purpose (daytona)
-<a href="http://www.hpl.hp.com/hosted/sortbenchmark/">terabyte sort
-benchmark</a>.
-
-<p>
-
-The cluster statistics were:
-<ul>
-<li> 910 nodes
-<li> 4 dual core Xeons @ 2.0ghz per a node
-<li> 4 SATA disks per a node
-<li> 8G RAM per a node
-<li> 1 gigabit ethernet on each node
-<li> 40 nodes per a rack
-<li> 8 gigabit ethernet uplinks from each rack to the core
-<li> Red Hat Enterprise Linux Server Release 5.1 (kernel 2.6.18)
-<li> Sun Java JDK 1.6.0_05-b13
-</ul>
-
-<p>
-
-The test was on Hadoop trunk (pre-0.18) patched with <a
-href="http://issues.apache.org/jira/browse/HADOOP-3443">HADOOP-3443</a>
-and <a
-href="http://issues.apache.org/jira/browse/HADOOP-3446">HADOOP-3446</a>,
-which were required to remove intermediate writes to disk.
-TeraGen used
-1800 tasks to generate a total of 10 billion rows in HDFS, with a
-block size of 1024 MB.
-TeraSort was configured with 1800 maps and 1800 reduces, and
-<i>mapreduce.task.io.sort.mb</i>,
-<i>mapreduce.task.io.sort.factor</i>,
-<i>fs.inmemory.size.mb</i>, and task heap size
-sufficient that transient data was never spilled to disk, other at the
-end of the map. The sampler looked at 100,000 keys to determine the
-reduce boundaries, which lead to imperfect balancing with reduce
-outputs ranging from 337 MB to 872 MB.
-
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/processor/FilterByWordInputProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/processor/FilterByWordInputProcessor.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/processor/FilterByWordInputProcessor.java
deleted file mode 100644
index aa7b836..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/processor/FilterByWordInputProcessor.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.processor;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.mapreduce.examples.FilterLinesByWord;
-import org.apache.tez.mapreduce.examples.FilterLinesByWord.TextLongPair;
-import org.apache.tez.mapreduce.hadoop.MRJobConfig;
-import org.apache.tez.mapreduce.input.MRInput;
-import org.apache.tez.mapreduce.input.MRInputLegacy;
-import org.apache.tez.runtime.api.AbstractLogicalIOProcessor;
-import org.apache.tez.runtime.api.Event;
-import org.apache.tez.runtime.api.LogicalInput;
-import org.apache.tez.runtime.api.LogicalOutput;
-import org.apache.tez.runtime.api.ProcessorContext;
-import org.apache.tez.runtime.library.api.KeyValueReader;
-import org.apache.tez.runtime.library.api.KeyValueWriter;
-import org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput;
-
-public class FilterByWordInputProcessor extends AbstractLogicalIOProcessor {
-
- private static final Log LOG = LogFactory.getLog(FilterByWordInputProcessor.class);
-
- private String filterWord;
-
- public FilterByWordInputProcessor(ProcessorContext context) {
- super(context);
- }
-
-
- @Override
- public void initialize() throws Exception {
- Configuration conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());
- filterWord = conf.get(FilterLinesByWord.FILTER_PARAM_NAME);
- if (filterWord == null) {
- getContext().fatalError(null, "No filter word specified");
- }
- }
-
- @Override
- public void handleEvents(List<Event> processorEvents) {
- throw new UnsupportedOperationException("Not expecting any events to the broadcast processor");
-
- }
-
- @Override
- public void close() throws Exception {
- LOG.info("Broadcast Processor closing. Nothing to do");
- }
-
- @Override
- public void run(Map<String, LogicalInput> inputs,
- Map<String, LogicalOutput> outputs) throws Exception {
-
- if (inputs.size() != 1) {
- throw new IllegalStateException("FilterByWordInputProcessor processor can only work with a single input");
- }
-
- if (outputs.size() != 1) {
- throw new IllegalStateException("FilterByWordInputProcessor processor can only work with a single output");
- }
-
- for (LogicalInput input : inputs.values()) {
- input.start();
- }
- for (LogicalOutput output : outputs.values()) {
- output.start();
- }
-
- LogicalInput li = inputs.values().iterator().next();
- if (! (li instanceof MRInput)) {
- throw new IllegalStateException("FilterByWordInputProcessor processor can only work with MRInput");
- }
-
- LogicalOutput lo = outputs.values().iterator().next();
- if (! (lo instanceof OnFileUnorderedKVOutput)) {
- throw new IllegalStateException("FilterByWordInputProcessor processor can only work with OnFileUnorderedKVOutput");
- }
-
- MRInputLegacy mrInput = (MRInputLegacy) li;
- mrInput.init();
- OnFileUnorderedKVOutput kvOutput = (OnFileUnorderedKVOutput) lo;
-
- Configuration updatedConf = mrInput.getConfigUpdates();
- Text srcFile = new Text();
- srcFile.set("UNKNOWN_FILENAME_IN_PROCESSOR");
- if (updatedConf != null) {
- String fileName = updatedConf.get(MRJobConfig.MAP_INPUT_FILE);
- if (fileName != null) {
- LOG.info("Processing file: " + fileName);
- srcFile.set(fileName);
- }
- }
-
- KeyValueReader kvReader = mrInput.getReader();
- KeyValueWriter kvWriter = kvOutput.getWriter();
-
- while (kvReader.next()) {
- Object key = kvReader.getCurrentKey();
- Object val = kvReader.getCurrentValue();
-
- Text valText = (Text) val;
- String readVal = valText.toString();
- if (readVal.contains(filterWord)) {
- LongWritable lineNum = (LongWritable) key;
- TextLongPair outVal = new TextLongPair(srcFile, lineNum);
- kvWriter.write(valText, outVal);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/processor/FilterByWordOutputProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/processor/FilterByWordOutputProcessor.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/processor/FilterByWordOutputProcessor.java
deleted file mode 100644
index 9eca13f..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/processor/FilterByWordOutputProcessor.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.processor;
-
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.tez.mapreduce.output.MROutput;
-import org.apache.tez.mapreduce.processor.SimpleMRProcessor;
-import org.apache.tez.mapreduce.processor.map.MapProcessor;
-import org.apache.tez.runtime.api.Event;
-import org.apache.tez.runtime.api.LogicalInput;
-import org.apache.tez.runtime.api.LogicalOutput;
-import org.apache.tez.runtime.api.ProcessorContext;
-import org.apache.tez.runtime.library.api.KeyValueReader;
-import org.apache.tez.runtime.library.api.KeyValueWriter;
-import org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput;
-
-
-public class FilterByWordOutputProcessor extends SimpleMRProcessor {
-
- private static final Log LOG = LogFactory.getLog(MapProcessor.class);
-
- public FilterByWordOutputProcessor(ProcessorContext context) {
- super(context);
- }
-
-
- @Override
- public void handleEvents(List<Event> processorEvents) {
- throw new UnsupportedOperationException("Not expecting any events to the broadcast output processor");
- }
-
- @Override
- public void close() throws Exception {
- LOG.info("Broadcast Output Processor closing. Nothing to do");
- }
-
- @Override
- public void run() throws Exception {
-
- if (inputs.size() != 1) {
- throw new IllegalStateException("FilterByWordOutputProcessor processor can only work with a single input");
- }
-
- if (outputs.size() != 1) {
- throw new IllegalStateException("FilterByWordOutputProcessor processor can only work with a single output");
- }
-
- for (LogicalInput input : inputs.values()) {
- input.start();
- }
- for (LogicalOutput output : outputs.values()) {
- output.start();
- }
-
- LogicalInput li = inputs.values().iterator().next();
- if (! (li instanceof ShuffledUnorderedKVInput)) {
- throw new IllegalStateException("FilterByWordOutputProcessor processor can only work with ShuffledUnorderedKVInput");
- }
-
- LogicalOutput lo = outputs.values().iterator().next();
- if (! (lo instanceof MROutput)) {
- throw new IllegalStateException("FilterByWordOutputProcessor processor can only work with MROutput");
- }
-
- ShuffledUnorderedKVInput kvInput = (ShuffledUnorderedKVInput) li;
- MROutput mrOutput = (MROutput) lo;
-
- KeyValueReader kvReader = kvInput.getReader();
- KeyValueWriter kvWriter = mrOutput.getWriter();
- while (kvReader.next()) {
- Object key = kvReader.getCurrentKey();
- Object value = kvReader.getCurrentValue();
-
- kvWriter.write(key, value);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/test/java/org/apache/tez/mapreduce/examples/terasort/TestTeraSort.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/test/java/org/apache/tez/mapreduce/examples/terasort/TestTeraSort.java b/tez-mapreduce-examples/src/test/java/org/apache/tez/mapreduce/examples/terasort/TestTeraSort.java
deleted file mode 100644
index 0d4f05c..0000000
--- a/tez-mapreduce-examples/src/test/java/org/apache/tez/mapreduce/examples/terasort/TestTeraSort.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tez.mapreduce.examples.terasort;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.HadoopTestCase;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.tez.mapreduce.examples.terasort.TeraGen;
-import org.apache.tez.mapreduce.examples.terasort.TeraSort;
-import org.apache.tez.mapreduce.examples.terasort.TeraValidate;
-import org.junit.Ignore;
-@Ignore
-public class TestTeraSort extends HadoopTestCase {
-
- public TestTeraSort()
- throws IOException {
- super(CLUSTER_MR, DFS_FS, 1, 1);
- }
-
- protected void tearDown() throws Exception {
- getFileSystem().delete(new Path(TEST_DIR), true);
- super.tearDown();
- }
-
- // Input/Output paths for sort
- private static final String TEST_DIR =
- new File(System.getProperty("test.build.data", "/tmp"), "terasort")
- .getAbsolutePath();
- private static final Path SORT_INPUT_PATH = new Path(TEST_DIR, "sortin");
- private static final Path SORT_OUTPUT_PATH = new Path(TEST_DIR, "sortout");
- private static final Path TERA_OUTPUT_PATH = new Path(TEST_DIR, "validate");
- private static final String NUM_ROWS = "100";
-
- private void runTeraGen(Configuration conf, Path sortInput)
- throws Exception {
- String[] genArgs = {NUM_ROWS, sortInput.toString()};
-
- // Run TeraGen
- assertEquals(ToolRunner.run(conf, new TeraGen(), genArgs), 0);
- }
-
- private void runTeraSort(Configuration conf,
- Path sortInput, Path sortOutput) throws Exception {
-
- // Setup command-line arguments to 'sort'
- String[] sortArgs = {sortInput.toString(), sortOutput.toString()};
-
- // Run Sort
- assertEquals(ToolRunner.run(conf, new TeraSort(), sortArgs), 0);
- }
-
- private void runTeraValidator(Configuration job,
- Path sortOutput, Path valOutput)
- throws Exception {
- String[] svArgs = {sortOutput.toString(), valOutput.toString()};
-
- // Run Tera-Validator
- assertEquals(ToolRunner.run(job, new TeraValidate(), svArgs), 0);
- }
-
- public void testTeraSort() throws Exception {
- // Run TeraGen to generate input for 'terasort'
- runTeraGen(createJobConf(), SORT_INPUT_PATH);
-
- // Run terasort
- runTeraSort(createJobConf(), SORT_INPUT_PATH, SORT_OUTPUT_PATH);
-
- // Run tera-validator to check if sort worked correctly
- runTeraValidator(createJobConf(), SORT_OUTPUT_PATH,
- TERA_OUTPUT_PATH);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/pom.xml
----------------------------------------------------------------------
diff --git a/tez-tests/pom.xml b/tez-tests/pom.xml
index f5282dc..6972372 100644
--- a/tez-tests/pom.xml
+++ b/tez-tests/pom.xml
@@ -28,7 +28,6 @@
<dependency>
<groupId>org.apache.tez</groupId>
<artifactId>tez-api</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.tez</groupId>
@@ -38,16 +37,14 @@
<dependency>
<groupId>org.apache.tez</groupId>
<artifactId>tez-runtime-library</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.tez</groupId>
<artifactId>tez-mapreduce</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.tez</groupId>
- <artifactId>tez-mapreduce-examples</artifactId>
+ <artifactId>tez-examples</artifactId>
<scope>test</scope>
</dependency>
<dependency>
@@ -58,7 +55,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -75,17 +71,14 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -96,12 +89,10 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-common</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -112,12 +103,10 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
@@ -129,16 +118,37 @@
<build>
<plugins>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifest>
- <mainClass>org.apache.tez.test.TestDriver</mainClass>
- </manifest>
- </archive>
- </configuration>
- </plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>default-jar</id>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <configuration>
+ <archive>
+ <manifest>
+ <mainClass>org.apache.tez.mapreduce.examples.ExampleDriver</mainClass>
+ </manifest>
+ </archive>
+ </configuration>
+ </execution>
+ <execution>
+ <id>default</id>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ <configuration>
+ <archive>
+ <manifest>
+ <mainClass>org.apache.tez.test.TestDriver</mainClass>
+ </manifest>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
new file mode 100644
index 0000000..35a7c76
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
@@ -0,0 +1,259 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.tez.client.TezClient;
+import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.Edge;
+import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.VertexManagerPluginDescriptor;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.library.vertexmanager.InputReadyVertexManager;
+import org.apache.tez.runtime.api.ObjectRegistry;
+import org.apache.tez.runtime.api.ProcessorContext;
+import org.apache.tez.runtime.library.api.KeyValueReader;
+import org.apache.tez.runtime.library.api.KeyValueWriter;
+import org.apache.tez.runtime.library.conf.UnorderedUnpartitionedKVEdgeConfigurer;
+import org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput;
+import org.apache.tez.runtime.library.processor.SimpleProcessor;
+
+import com.google.common.base.Preconditions;
+
+public class BroadcastAndOneToOneExample extends Configured implements Tool {
+ public static class InputProcessor extends SimpleProcessor {
+ Text word = new Text();
+
+ public InputProcessor(ProcessorContext context) {
+ super(context);
+ }
+
+ @Override
+ public void run() throws Exception {
+ Preconditions.checkArgument(getOutputs().size() == 1);
+ OnFileUnorderedKVOutput output = (OnFileUnorderedKVOutput) getOutputs().values().iterator()
+ .next();
+ KeyValueWriter kvWriter = (KeyValueWriter) output.getWriter();
+ kvWriter.write(word, new IntWritable(getContext().getTaskIndex()));
+ byte[] userPayload = getContext().getUserPayload().getPayload();
+ if (userPayload != null) {
+ boolean doLocalityCheck = userPayload[0] > 0 ? true : false;
+ if (doLocalityCheck) {
+ ObjectRegistry objectRegistry = getContext().getObjectRegistry();
+ String entry = String.valueOf(getContext().getTaskIndex());
+ objectRegistry.cacheForDAG(entry, entry);
+ }
+ }
+ }
+ }
+
+ public static class OneToOneProcessor extends SimpleProcessor {
+ Text word = new Text();
+
+ public OneToOneProcessor(ProcessorContext context) {
+ super(context);
+ }
+
+ @Override
+ public void run() throws Exception {
+ Preconditions.checkArgument(inputs.size() == 2);
+
+ KeyValueReader inputKvReader = (KeyValueReader) getInputs().get("Input").getReader();
+ KeyValueReader broadcastKvReader = (KeyValueReader) getInputs().get("Broadcast").getReader();
+ int sum = 0;
+ while (broadcastKvReader.next()) {
+ sum += ((IntWritable) broadcastKvReader.getCurrentValue()).get();
+ }
+ while (inputKvReader.next()) {
+ sum += ((IntWritable) inputKvReader.getCurrentValue()).get();
+ }
+ boolean doLocalityCheck = getContext().getUserPayload().getPayload()[0] > 0 ? true : false;
+ int broadcastSum = getContext().getUserPayload().getPayload()[1];
+ int expectedSum = broadcastSum + getContext().getTaskIndex();
+ System.out.println("Index: " + getContext().getTaskIndex() +
+ " sum: " + sum + " expectedSum: " + expectedSum + " broadcastSum: " + broadcastSum);
+ Preconditions.checkState((sum == expectedSum), "Sum = " + sum);
+
+ if (doLocalityCheck) {
+ ObjectRegistry objectRegistry = getContext().getObjectRegistry();
+ String index = (String) objectRegistry.get(String.valueOf(getContext().getTaskIndex()));
+ if (index == null || Integer.valueOf(index).intValue() != getContext().getTaskIndex()) {
+ String msg = "Did not find expected local producer "
+ + getContext().getTaskIndex() + " in the same JVM";
+ System.out.println(msg);
+ throw new TezUncheckedException(msg);
+ }
+ }
+ }
+
+ }
+
+ private DAG createDAG(FileSystem fs, TezConfiguration tezConf,
+ Path stagingDir, boolean doLocalityCheck) throws IOException, YarnException {
+
+ int numBroadcastTasks = 2;
+ int numOneToOneTasks = 3;
+ if (doLocalityCheck) {
+ YarnClient yarnClient = YarnClient.createYarnClient();
+ yarnClient.init(tezConf);
+ yarnClient.start();
+ int numNMs = yarnClient.getNodeReports(NodeState.RUNNING).size();
+ yarnClient.stop();
+ // create enough 1-1 tasks to run in parallel
+ numOneToOneTasks = numNMs - numBroadcastTasks - 1;// 1 AM
+ if (numOneToOneTasks < 1) {
+ numOneToOneTasks = 1;
+ }
+ }
+ byte[] procByte = {(byte) (doLocalityCheck ? 1 : 0), 1};
+ UserPayload procPayload = new UserPayload(procByte);
+
+ System.out.println("Using " + numOneToOneTasks + " 1-1 tasks");
+
+ Vertex broadcastVertex = new Vertex("Broadcast", new ProcessorDescriptor(
+ InputProcessor.class.getName()), numBroadcastTasks);
+
+ Vertex inputVertex = new Vertex("Input", new ProcessorDescriptor(
+ InputProcessor.class.getName()).setUserPayload(procPayload), numOneToOneTasks);
+
+ Vertex oneToOneVertex = new Vertex("OneToOne",
+ new ProcessorDescriptor(
+ OneToOneProcessor.class.getName()).setUserPayload(procPayload));
+ oneToOneVertex.setVertexManagerPlugin(
+ new VertexManagerPluginDescriptor(InputReadyVertexManager.class.getName()));
+
+ UnorderedUnpartitionedKVEdgeConfigurer edgeConf = UnorderedUnpartitionedKVEdgeConfigurer
+ .newBuilder(Text.class.getName(), IntWritable.class.getName()).build();
+
+ DAG dag = new DAG("BroadcastAndOneToOneExample");
+ dag.addVertex(inputVertex)
+ .addVertex(broadcastVertex)
+ .addVertex(oneToOneVertex)
+ .addEdge(
+ new Edge(inputVertex, oneToOneVertex, edgeConf.createDefaultOneToOneEdgeProperty()))
+ .addEdge(
+ new Edge(broadcastVertex, oneToOneVertex,
+ edgeConf.createDefaultBroadcastEdgeProperty()));
+ return dag;
+ }
+
+ public boolean run(Configuration conf, boolean doLocalityCheck) throws Exception {
+ System.out.println("Running BroadcastAndOneToOneExample");
+ // conf and UGI
+ TezConfiguration tezConf;
+ if (conf != null) {
+ tezConf = new TezConfiguration(conf);
+ } else {
+ tezConf = new TezConfiguration();
+ }
+ tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true);
+ UserGroupInformation.setConfiguration(tezConf);
+ String user = UserGroupInformation.getCurrentUser().getShortUserName();
+
+ // staging dir
+ FileSystem fs = FileSystem.get(tezConf);
+ String stagingDirStr = Path.SEPARATOR + "user" + Path.SEPARATOR
+ + user + Path.SEPARATOR+ ".staging" + Path.SEPARATOR
+ + Path.SEPARATOR + Long.toString(System.currentTimeMillis());
+ Path stagingDir = new Path(stagingDirStr);
+ tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr);
+ stagingDir = fs.makeQualified(stagingDir);
+
+ // No need to add jar containing this class as assumed to be part of
+ // the tez jars.
+
+ // TEZ-674 Obtain tokens based on the Input / Output paths. For now assuming staging dir
+ // is the same filesystem as the one used for Input/Output.
+ TezClient tezSession = null;
+ // needs session or else TaskScheduler does not hold onto containers
+ tezSession = new TezClient("broadcastAndOneToOneExample", tezConf);
+ tezSession.start();
+
+ DAGClient dagClient = null;
+
+ try {
+ DAG dag = createDAG(fs, tezConf, stagingDir, doLocalityCheck);
+
+ tezSession.waitTillReady();
+ dagClient = tezSession.submitDAG(dag);
+
+ // monitoring
+ DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(null);
+ if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
+ System.out.println("DAG diagnostics: " + dagStatus.getDiagnostics());
+ return false;
+ }
+ return true;
+ } finally {
+ fs.delete(stagingDir, true);
+ tezSession.stop();
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ boolean doLocalityCheck = true;
+ if (args.length == 1) {
+ if (args[0].equals(skipLocalityCheck)) {
+ doLocalityCheck = false;
+ } else {
+ printUsage();
+ throw new TezException("Invalid command line");
+ }
+ } else if (args.length > 1) {
+ printUsage();
+ throw new TezException("Invalid command line");
+ }
+ boolean status = run(getConf(), doLocalityCheck);
+ return status ? 0 : 1;
+ }
+
+ private static void printUsage() {
+ System.err.println("broadcastAndOneToOneExample " + skipLocalityCheck);
+ ToolRunner.printGenericCommandUsage(System.err);
+ }
+
+ static String skipLocalityCheck = "-skipLocalityCheck";
+
+ public static void main(String[] args) throws Exception {
+ Configuration conf = new Configuration();
+ BroadcastAndOneToOneExample job = new BroadcastAndOneToOneExample();
+ int status = ToolRunner.run(conf, job, args);
+ System.exit(status);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
new file mode 100644
index 0000000..bfdcc44
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples;
+
+import java.io.IOException;
+import java.text.DecimalFormat;
+import java.util.EnumSet;
+import java.util.Set;
+
+import org.apache.hadoop.util.ProgramDriver;
+import org.apache.tez.common.counters.TezCounters;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.api.client.Progress;
+import org.apache.tez.dag.api.client.StatusGetOpts;
+import org.apache.tez.dag.api.client.VertexStatus;
+import org.apache.tez.mapreduce.examples.terasort.TeraGen;
+import org.apache.tez.mapreduce.examples.terasort.TeraSort;
+import org.apache.tez.mapreduce.examples.terasort.TeraValidate;
+
+/**
+ * A description of an example program based on its class and a
+ * human-readable description.
+ */
+public class ExampleDriver {
+
+ private static final DecimalFormat formatter = new DecimalFormat("###.##%");
+
+ public static void main(String argv[]){
+ int exitCode = -1;
+ ProgramDriver pgd = new ProgramDriver();
+ try {
+ pgd.addClass("wordcount", MapredWordCount.class,
+ "A map/reduce program that counts the words in the input files.");
+ pgd.addClass("mapredwordcount", MapredWordCount.class,
+ "A map/reduce program that counts the words in the input files"
+ + " using the mapred apis.");
+ pgd.addClass("randomwriter", RandomWriter.class,
+ "A map/reduce program that writes 10GB of random data per node.");
+ pgd.addClass("randomtextwriter", RandomTextWriter.class,
+ "A map/reduce program that writes 10GB of random textual data per node.");
+ pgd.addClass("sort", Sort.class,
+ "A map/reduce program that sorts the data written by the random"
+ + " writer.");
+ pgd.addClass("secondarysort", SecondarySort.class,
+ "An example defining a secondary sort to the reduce.");
+ pgd.addClass("join", Join.class,
+ "A job that effects a join over sorted, equally partitioned"
+ + " datasets");
+ pgd.addClass("teragen", TeraGen.class,
+ "Generate data for the terasort");
+ pgd.addClass("terasort", TeraSort.class,
+ "Run the terasort");
+ pgd.addClass("teravalidate", TeraValidate.class,
+ "Checking results of terasort");
+ pgd.addClass("groupbyorderbymrrtest", GroupByOrderByMRRTest.class,
+ "A map-reduce-reduce program that does groupby-order by. Takes input"
+ + " containing employee_name department name per line of input"
+ + " and generates count of employees per department and"
+ + " sorted on employee count");
+ pgd.addClass("mrrsleep", MRRSleepJob.class,
+ "MRR Sleep Job");
+ pgd.addClass("testorderedwordcount", TestOrderedWordCount.class,
+ "Word Count with words sorted on frequency");
+ pgd.addClass("unionexample", UnionExample.class,
+ "Union example");
+ pgd.addClass("broadcastAndOneToOneExample", BroadcastAndOneToOneExample.class,
+ "BroadcastAndOneToOneExample example");
+ pgd.addClass("filterLinesByWord", FilterLinesByWord.class,
+ "Filters lines by the specified word using broadcast edge");
+ pgd.addClass("filterLinesByWordOneToOne", FilterLinesByWordOneToOne.class,
+ "Filters lines by the specified word using OneToOne edge");
+ pgd.addClass("intersect", IntersectExample.class,
+ "Identify all occurences of lines in file1 which also occur in file2");
+ pgd.addClass("intersectdatagen", IntersectDataGen.class,
+ "Generate data to run the intersect example");
+ pgd.addClass("intersectvalidate", IntersectValidate.class,
+ "Validate data generated by intersect and intersectdatagen");
+ exitCode = pgd.run(argv);
+ }
+ catch(Throwable e){
+ e.printStackTrace();
+ }
+
+ System.exit(exitCode);
+ }
+
+ public static void printDAGStatus(DAGClient dagClient, String[] vertexNames)
+ throws IOException, TezException {
+ printDAGStatus(dagClient, vertexNames, false, false);
+ }
+
+ public static void printDAGStatus(DAGClient dagClient, String[] vertexNames,
+ boolean displayDAGCounters, boolean displayVertexCounters)
+ throws IOException, TezException {
+ Set<StatusGetOpts> opts = EnumSet.of(StatusGetOpts.GET_COUNTERS);
+ DAGStatus dagStatus = dagClient.getDAGStatus(
+ (displayDAGCounters ? opts : null));
+ Progress progress = dagStatus.getDAGProgress();
+ double vProgressFloat = 0.0f;
+ if (progress != null) {
+ System.out.println("");
+ System.out.println("DAG: State: "
+ + dagStatus.getState()
+ + " Progress: "
+ + (progress.getTotalTaskCount() < 0 ? formatter.format(0.0f) :
+ formatter.format((double)(progress.getSucceededTaskCount())
+ /progress.getTotalTaskCount())));
+ for (String vertexName : vertexNames) {
+ VertexStatus vStatus = dagClient.getVertexStatus(vertexName,
+ (displayVertexCounters ? opts : null));
+ if (vStatus == null) {
+ System.out.println("Could not retrieve status for vertex: "
+ + vertexName);
+ continue;
+ }
+ Progress vProgress = vStatus.getProgress();
+ if (vProgress != null) {
+ vProgressFloat = 0.0f;
+ if (vProgress.getTotalTaskCount() == 0) {
+ vProgressFloat = 1.0f;
+ } else if (vProgress.getTotalTaskCount() > 0) {
+ vProgressFloat = (double)vProgress.getSucceededTaskCount()
+ /vProgress.getTotalTaskCount();
+ }
+ System.out.println("VertexStatus:"
+ + " VertexName: "
+ + (vertexName.equals("ivertex1") ? "intermediate-reducer"
+ : vertexName)
+ + " Progress: " + formatter.format(vProgressFloat));
+ }
+ if (displayVertexCounters) {
+ TezCounters counters = vStatus.getVertexCounters();
+ if (counters != null) {
+ System.out.println("Vertex Counters for " + vertexName + ": "
+ + counters);
+ }
+ }
+ }
+ }
+ if (displayDAGCounters) {
+ TezCounters counters = dagStatus.getDAGCounters();
+ if (counters != null) {
+ System.out.println("DAG Counters: " + counters);
+ }
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
new file mode 100644
index 0000000..02a3202
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
@@ -0,0 +1,297 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.UUID;
+
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.security.TokenCache;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.util.ClassUtil;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.client.TezClientUtils;
+import org.apache.tez.client.TezClient;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.DataSinkDescriptor;
+import org.apache.tez.dag.api.DataSourceDescriptor;
+import org.apache.tez.dag.api.Edge;
+import org.apache.tez.dag.api.OutputCommitterDescriptor;
+import org.apache.tez.dag.api.OutputDescriptor;
+import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.api.client.StatusGetOpts;
+import org.apache.tez.mapreduce.committer.MROutputCommitter;
+import org.apache.tez.mapreduce.examples.helpers.SplitsInClientOptionParser;
+import org.apache.tez.mapreduce.hadoop.MRInputHelpers;
+import org.apache.tez.mapreduce.input.MRInputLegacy;
+import org.apache.tez.mapreduce.output.MROutput;
+import org.apache.tez.processor.FilterByWordInputProcessor;
+import org.apache.tez.processor.FilterByWordOutputProcessor;
+import org.apache.tez.runtime.library.conf.UnorderedUnpartitionedKVEdgeConfigurer;
+
+import com.google.common.collect.Sets;
+
+public class FilterLinesByWord extends Configured implements Tool {
+
+ private static Log LOG = LogFactory.getLog(FilterLinesByWord.class);
+
+ public static final String FILTER_PARAM_NAME = "tez.runtime.examples.filterbyword.word";
+
+ private boolean exitOnCompletion = false;
+
+ public FilterLinesByWord(boolean exitOnCompletion) {
+ this.exitOnCompletion = exitOnCompletion;
+ }
+
+ private static void printUsage() {
+ System.err.println("Usage filtelinesrbyword <in> <out> <filter_word> [-generateSplitsInClient true/<false>]");
+ ToolRunner.printGenericCommandUsage(System.err);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Configuration conf = getConf();
+ String [] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+ Credentials credentials = new Credentials();
+
+ boolean generateSplitsInClient = false;
+
+ SplitsInClientOptionParser splitCmdLineParser = new SplitsInClientOptionParser();
+ try {
+ generateSplitsInClient = splitCmdLineParser.parse(otherArgs, false);
+ otherArgs = splitCmdLineParser.getRemainingArgs();
+ } catch (ParseException e1) {
+ System.err.println("Invalid options");
+ printUsage();
+ return 2;
+ }
+
+ if (otherArgs.length != 3) {
+ printUsage();
+ return 2;
+ }
+
+ String inputPath = otherArgs[0];
+ String outputPath = otherArgs[1];
+ String filterWord = otherArgs[2];
+
+ FileSystem fs = FileSystem.get(conf);
+ if (fs.exists(new Path(outputPath))) {
+ System.err.println("Output directory : " + outputPath + " already exists");
+ return 2;
+ }
+
+ TezConfiguration tezConf = new TezConfiguration(conf);
+
+ fs.getWorkingDirectory();
+ Path stagingDir = new Path(fs.getWorkingDirectory(), UUID.randomUUID().toString());
+ tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());
+ TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
+
+ String jarPath = ClassUtil.findContainingJar(FilterLinesByWord.class);
+ if (jarPath == null) {
+ throw new TezUncheckedException("Could not find any jar containing"
+ + FilterLinesByWord.class.getName() + " in the classpath");
+ }
+
+ Path remoteJarPath = fs.makeQualified(new Path(stagingDir, "dag_job.jar"));
+ fs.copyFromLocalFile(new Path(jarPath), remoteJarPath);
+ FileStatus remoteJarStatus = fs.getFileStatus(remoteJarPath);
+ TokenCache.obtainTokensForNamenodes(credentials, new Path[]{remoteJarPath}, conf);
+
+ Map<String, LocalResource> commonLocalResources = new TreeMap<String, LocalResource>();
+ LocalResource dagJarLocalRsrc = LocalResource.newInstance(
+ ConverterUtils.getYarnUrlFromPath(remoteJarPath),
+ LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
+ remoteJarStatus.getLen(), remoteJarStatus.getModificationTime());
+ commonLocalResources.put("dag_job.jar", dagJarLocalRsrc);
+
+
+
+ TezClient tezSession = new TezClient("FilterLinesByWordSession", tezConf,
+ commonLocalResources, credentials);
+ tezSession.start(); // Why do I need to start the TezSession.
+
+ Configuration stage1Conf = new JobConf(conf);
+ stage1Conf.set(FILTER_PARAM_NAME, filterWord);
+
+ Configuration stage2Conf = new JobConf(conf);
+ stage2Conf.set(FileOutputFormat.OUTDIR, outputPath);
+ stage2Conf.setBoolean("mapred.mapper.new-api", false);
+
+ UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf);
+ // Setup stage1 Vertex
+ Vertex stage1Vertex = new Vertex("stage1", new ProcessorDescriptor(
+ FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload))
+ .setTaskLocalFiles(commonLocalResources);
+
+ DataSourceDescriptor dsd;
+ if (generateSplitsInClient) {
+ // TODO TEZ-1406. Dont' use MRInputLegacy
+ stage1Conf.set(FileInputFormat.INPUT_DIR, inputPath);
+ stage1Conf.setBoolean("mapred.mapper.new-api", false);
+ dsd = MRInputHelpers.configureMRInputWithLegacySplitGeneration(stage1Conf, stagingDir, true);
+ } else {
+ dsd = MRInputLegacy.createConfigurer(stage1Conf, TextInputFormat.class, inputPath)
+ .groupSplits(false).create();
+ }
+ stage1Vertex.addDataSource("MRInput", dsd);
+
+ // Setup stage2 Vertex
+ Vertex stage2Vertex = new Vertex("stage2", new ProcessorDescriptor(
+ FilterByWordOutputProcessor.class.getName()).setUserPayload(
+ TezUtils.createUserPayloadFromConf(stage2Conf)), 1);
+ stage2Vertex.setTaskLocalFiles(commonLocalResources);
+
+ // Configure the Output for stage2
+ OutputDescriptor od = new OutputDescriptor(MROutput.class.getName())
+ .setUserPayload(TezUtils.createUserPayloadFromConf(stage2Conf));
+ OutputCommitterDescriptor ocd = new OutputCommitterDescriptor(MROutputCommitter.class.getName());
+ stage2Vertex.addDataSink("MROutput", new DataSinkDescriptor(od, ocd, null));
+
+ UnorderedUnpartitionedKVEdgeConfigurer edgeConf = UnorderedUnpartitionedKVEdgeConfigurer
+ .newBuilder(Text.class.getName(), TextLongPair.class.getName()).build();
+
+ DAG dag = new DAG("FilterLinesByWord");
+ Edge edge = new Edge(stage1Vertex, stage2Vertex, edgeConf.createDefaultBroadcastEdgeProperty());
+ dag.addVertex(stage1Vertex).addVertex(stage2Vertex).addEdge(edge);
+
+ LOG.info("Submitting DAG to Tez Session");
+ DAGClient dagClient = tezSession.submitDAG(dag);
+ LOG.info("Submitted DAG to Tez Session");
+
+ DAGStatus dagStatus = null;
+ String[] vNames = { "stage1", "stage2" };
+ try {
+ while (true) {
+ dagStatus = dagClient.getDAGStatus(null);
+ if(dagStatus.getState() == DAGStatus.State.RUNNING ||
+ dagStatus.getState() == DAGStatus.State.SUCCEEDED ||
+ dagStatus.getState() == DAGStatus.State.FAILED ||
+ dagStatus.getState() == DAGStatus.State.KILLED ||
+ dagStatus.getState() == DAGStatus.State.ERROR) {
+ break;
+ }
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ // continue;
+ }
+ }
+
+ while (dagStatus.getState() == DAGStatus.State.RUNNING) {
+ try {
+ ExampleDriver.printDAGStatus(dagClient, vNames);
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ // continue;
+ }
+ dagStatus = dagClient.getDAGStatus(null);
+ } catch (TezException e) {
+ LOG.fatal("Failed to get application progress. Exiting");
+ return -1;
+ }
+ }
+
+ dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
+
+ } finally {
+ fs.delete(stagingDir, true);
+ tezSession.stop();
+ }
+
+ ExampleDriver.printDAGStatus(dagClient, vNames, true, true);
+ LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
+ return dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1;
+ }
+
+ public static void main(String[] args) throws Exception {
+ FilterLinesByWord fl = new FilterLinesByWord(true);
+ int status = ToolRunner.run(new Configuration(), fl, args);
+ if (fl.exitOnCompletion) {
+ System.exit(status);
+ }
+ }
+
+ public static class TextLongPair implements Writable {
+
+ private Text text;
+ private LongWritable longWritable;
+
+ public TextLongPair() {
+ }
+
+ public TextLongPair(Text text, LongWritable longWritable) {
+ this.text = text;
+ this.longWritable = longWritable;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ this.text.write(out);
+ this.longWritable.write(out);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ this.text = new Text();
+ this.longWritable = new LongWritable();
+ text.readFields(in);
+ longWritable.readFields(in);
+ }
+
+ @Override
+ public String toString() {
+ return text.toString() + "\t" + longWritable.get();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
new file mode 100644
index 0000000..b413993
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples;
+
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.UUID;
+
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.ClassUtil;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.client.TezClientUtils;
+import org.apache.tez.client.TezClient;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.DataSinkDescriptor;
+import org.apache.tez.dag.api.DataSourceDescriptor;
+import org.apache.tez.dag.api.Edge;
+import org.apache.tez.dag.api.OutputCommitterDescriptor;
+import org.apache.tez.dag.api.OutputDescriptor;
+import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.mapreduce.committer.MROutputCommitter;
+import org.apache.tez.mapreduce.examples.FilterLinesByWord.TextLongPair;
+import org.apache.tez.mapreduce.examples.helpers.SplitsInClientOptionParser;
+import org.apache.tez.mapreduce.hadoop.MRInputHelpers;
+import org.apache.tez.mapreduce.input.MRInputLegacy;
+import org.apache.tez.mapreduce.output.MROutput;
+import org.apache.tez.processor.FilterByWordInputProcessor;
+import org.apache.tez.processor.FilterByWordOutputProcessor;
+import org.apache.tez.runtime.library.conf.UnorderedUnpartitionedKVEdgeConfigurer;
+
+public class FilterLinesByWordOneToOne extends Configured implements Tool {
+
+ private static Log LOG = LogFactory.getLog(FilterLinesByWordOneToOne.class);
+
+ public static final String FILTER_PARAM_NAME = "tez.runtime.examples.filterbyword.word";
+
+ private static void printUsage() {
+ System.err.println("Usage filterLinesByWordOneToOne <in> <out> <filter_word>"
+ + " [-generateSplitsInClient true/<false>]");
+ ToolRunner.printGenericCommandUsage(System.err);
+ }
+
+ public static void main(String[] args) throws Exception {
+ Configuration conf = new Configuration();
+ String[] otherArgs = new GenericOptionsParser(conf, args)
+ .getRemainingArgs();
+ int status = ToolRunner.run(conf, new FilterLinesByWordOneToOne(),
+ otherArgs);
+ System.exit(status);
+ }
+
+ @Override
+ public int run(String[] otherArgs) throws Exception {
+ boolean generateSplitsInClient = false;
+ SplitsInClientOptionParser splitCmdLineParser = new SplitsInClientOptionParser();
+ try {
+ generateSplitsInClient = splitCmdLineParser.parse(otherArgs, false);
+ otherArgs = splitCmdLineParser.getRemainingArgs();
+ } catch (ParseException e1) {
+ System.err.println("Invalid options");
+ printUsage();
+ return 2;
+ }
+
+ if (otherArgs.length != 3) {
+ printUsage();
+ return 2;
+ }
+
+ String inputPath = otherArgs[0];
+ String outputPath = otherArgs[1];
+ String filterWord = otherArgs[2];
+
+ Configuration conf = getConf();
+ FileSystem fs = FileSystem.get(conf);
+ if (fs.exists(new Path(outputPath))) {
+ System.err.println("Output directory : " + outputPath + " already exists");
+ return 2;
+ }
+
+ TezConfiguration tezConf = new TezConfiguration(conf);
+
+ fs.getWorkingDirectory();
+ Path stagingDir = new Path(fs.getWorkingDirectory(), UUID.randomUUID().toString());
+ tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());
+ TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
+
+ String jarPath = ClassUtil.findContainingJar(FilterLinesByWordOneToOne.class);
+ if (jarPath == null) {
+ throw new TezUncheckedException("Could not find any jar containing"
+ + FilterLinesByWordOneToOne.class.getName() + " in the classpath");
+ }
+
+ Path remoteJarPath = fs.makeQualified(new Path(stagingDir, "dag_job.jar"));
+ fs.copyFromLocalFile(new Path(jarPath), remoteJarPath);
+ FileStatus remoteJarStatus = fs.getFileStatus(remoteJarPath);
+
+ Map<String, LocalResource> commonLocalResources = new TreeMap<String, LocalResource>();
+ LocalResource dagJarLocalRsrc = LocalResource.newInstance(
+ ConverterUtils.getYarnUrlFromPath(remoteJarPath),
+ LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
+ remoteJarStatus.getLen(), remoteJarStatus.getModificationTime());
+ commonLocalResources.put("dag_job.jar", dagJarLocalRsrc);
+
+
+
+ TezClient tezSession = new TezClient("FilterLinesByWordSession", tezConf,
+ commonLocalResources, null);
+ tezSession.start(); // Why do I need to start the TezSession.
+
+ Configuration stage1Conf = new JobConf(conf);
+ stage1Conf.set(FILTER_PARAM_NAME, filterWord);
+
+ Configuration stage2Conf = new JobConf(conf);
+
+ stage2Conf.set(FileOutputFormat.OUTDIR, outputPath);
+ stage2Conf.setBoolean("mapred.mapper.new-api", false);
+
+ UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf);
+ // Setup stage1 Vertex
+ Vertex stage1Vertex = new Vertex("stage1", new ProcessorDescriptor(
+ FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload))
+ .setTaskLocalFiles(commonLocalResources);
+
+ DataSourceDescriptor dsd;
+ if (generateSplitsInClient) {
+ // TODO TEZ-1406. Dont' use MRInputLegacy
+ stage1Conf.set(FileInputFormat.INPUT_DIR, inputPath);
+ stage1Conf.setBoolean("mapred.mapper.new-api", false);
+ dsd = MRInputHelpers.configureMRInputWithLegacySplitGeneration(stage1Conf, stagingDir, true);
+ } else {
+ dsd = MRInputLegacy.createConfigurer(stage1Conf, TextInputFormat.class, inputPath)
+ .groupSplits(false).create();
+ }
+ stage1Vertex.addDataSource("MRInput", dsd);
+
+ // Setup stage2 Vertex
+ Vertex stage2Vertex = new Vertex("stage2", new ProcessorDescriptor(
+ FilterByWordOutputProcessor.class.getName()).setUserPayload(TezUtils
+ .createUserPayloadFromConf(stage2Conf)), dsd.getNumberOfShards());
+ stage2Vertex.setTaskLocalFiles(commonLocalResources);
+
+ // Configure the Output for stage2
+ stage2Vertex.addDataSink(
+ "MROutput",
+ new DataSinkDescriptor(new OutputDescriptor(MROutput.class.getName())
+ .setUserPayload(TezUtils.createUserPayloadFromConf(stage2Conf)),
+ new OutputCommitterDescriptor(MROutputCommitter.class.getName()), null));
+
+ UnorderedUnpartitionedKVEdgeConfigurer edgeConf = UnorderedUnpartitionedKVEdgeConfigurer
+ .newBuilder(Text.class.getName(), TextLongPair.class.getName()).build();
+
+ DAG dag = new DAG("FilterLinesByWord");
+ Edge edge = new Edge(stage1Vertex, stage2Vertex, edgeConf.createDefaultOneToOneEdgeProperty());
+ dag.addVertex(stage1Vertex).addVertex(stage2Vertex).addEdge(edge);
+
+ LOG.info("Submitting DAG to Tez Session");
+ DAGClient dagClient = tezSession.submitDAG(dag);
+ LOG.info("Submitted DAG to Tez Session");
+
+ DAGStatus dagStatus = null;
+ String[] vNames = { "stage1", "stage2" };
+ try {
+ while (true) {
+ dagStatus = dagClient.getDAGStatus(null);
+ if(dagStatus.getState() == DAGStatus.State.RUNNING ||
+ dagStatus.getState() == DAGStatus.State.SUCCEEDED ||
+ dagStatus.getState() == DAGStatus.State.FAILED ||
+ dagStatus.getState() == DAGStatus.State.KILLED ||
+ dagStatus.getState() == DAGStatus.State.ERROR) {
+ break;
+ }
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ // continue;
+ }
+ }
+
+ while (dagStatus.getState() == DAGStatus.State.RUNNING) {
+ try {
+ ExampleDriver.printDAGStatus(dagClient, vNames);
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ // continue;
+ }
+ dagStatus = dagClient.getDAGStatus(null);
+ } catch (TezException e) {
+ LOG.fatal("Failed to get application progress. Exiting");
+ return -1;
+ }
+ }
+ } finally {
+ fs.delete(stagingDir, true);
+ tezSession.stop();
+ }
+
+ ExampleDriver.printDAGStatus(dagClient, vNames);
+ LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
+ return dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1;
+ }
+}
[10/10] git commit: TEZ-1055. Rename tez-mapreduce-examples to
tez-examples (Hitesh Shah via bikas)
Posted by bi...@apache.org.
TEZ-1055. Rename tez-mapreduce-examples to tez-examples (Hitesh Shah via bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/41f5cd8a
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/41f5cd8a
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/41f5cd8a
Branch: refs/heads/master
Commit: 41f5cd8a2cef0bd78f8208881459c0165f4cb56d
Parents: 4a41ccf
Author: Bikas Saha <bi...@apache.org>
Authored: Fri Aug 15 17:54:53 2014 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Fri Aug 15 17:54:53 2014 -0700
----------------------------------------------------------------------
BUILDING.txt | 4 +-
CHANGES.txt | 2 +
INSTALL.txt | 14 +-
docs/src/site/apt/install_0_5_0.apt | 10 +-
pom.xml | 4 +-
tez-dist/pom.xml | 2 +-
tez-dist/src/main/assembly/tez-dist-minimal.xml | 3 -
tez-dist/src/main/assembly/tez-dist.xml | 3 -
tez-examples/findbugs-exclude.xml | 16 +
tez-examples/pom.xml | 100 +++
.../org/apache/tez/examples/ExampleDriver.java | 125 +++
.../apache/tez/examples/OrderedWordCount.java | 220 ++++++
.../tez/examples/SimpleSessionExample.java | 142 ++++
.../java/org/apache/tez/examples/WordCount.java | 254 +++++++
tez-mapreduce-examples/findbugs-exclude.xml | 16 -
tez-mapreduce-examples/pom.xml | 119 ---
.../apache/tez/examples/OrderedWordCount.java | 220 ------
.../tez/examples/SimpleSessionExample.java | 142 ----
.../java/org/apache/tez/examples/WordCount.java | 254 -------
.../examples/BroadcastAndOneToOneExample.java | 259 -------
.../tez/mapreduce/examples/ExampleDriver.java | 174 -----
.../mapreduce/examples/FilterLinesByWord.java | 297 --------
.../examples/FilterLinesByWordOneToOne.java | 245 ------
.../examples/GroupByOrderByMRRTest.java | 247 ------
.../mapreduce/examples/IntersectDataGen.java | 350 ---------
.../mapreduce/examples/IntersectExample.java | 281 -------
.../mapreduce/examples/IntersectValidate.java | 257 -------
.../org/apache/tez/mapreduce/examples/Join.java | 176 -----
.../tez/mapreduce/examples/MRRSleepJob.java | 757 -------------------
.../tez/mapreduce/examples/MapredWordCount.java | 164 ----
.../mapreduce/examples/RandomTextWriter.java | 757 -------------------
.../tez/mapreduce/examples/RandomWriter.java | 298 --------
.../tez/mapreduce/examples/SecondarySort.java | 248 ------
.../org/apache/tez/mapreduce/examples/Sort.java | 201 -----
.../examples/TestOrderedWordCount.java | 479 ------------
.../tez/mapreduce/examples/UnionExample.java | 295 --------
.../helpers/SplitsInClientOptionParser.java | 72 --
.../apache/tez/mapreduce/examples/package.html | 23 -
.../examples/terasort/2009-write-up/.gitignore | 20 -
.../terasort/2009-write-up/Yahoo2009.tex | 370 ---------
.../examples/terasort/2009-write-up/tera.bib | 31 -
.../mapreduce/examples/terasort/GenSort.java | 251 ------
.../mapreduce/examples/terasort/Random16.java | 374 ---------
.../examples/terasort/TeraChecksum.java | 103 ---
.../mapreduce/examples/terasort/TeraGen.java | 311 --------
.../examples/terasort/TeraInputFormat.java | 353 ---------
.../examples/terasort/TeraOutputFormat.java | 112 ---
.../examples/terasort/TeraScheduler.java | 253 -------
.../mapreduce/examples/terasort/TeraSort.java | 335 --------
.../examples/terasort/TeraValidate.java | 188 -----
.../mapreduce/examples/terasort/Unsigned16.java | 297 --------
.../examples/terasort/job_history_summary.py | 100 ---
.../mapreduce/examples/terasort/package.html | 114 ---
.../processor/FilterByWordInputProcessor.java | 135 ----
.../processor/FilterByWordOutputProcessor.java | 96 ---
.../examples/terasort/TestTeraSort.java | 92 ---
tez-tests/pom.xml | 54 +-
.../examples/BroadcastAndOneToOneExample.java | 259 +++++++
.../tez/mapreduce/examples/ExampleDriver.java | 167 ++++
.../mapreduce/examples/FilterLinesByWord.java | 297 ++++++++
.../examples/FilterLinesByWordOneToOne.java | 245 ++++++
.../examples/GroupByOrderByMRRTest.java | 247 ++++++
.../mapreduce/examples/IntersectDataGen.java | 350 +++++++++
.../mapreduce/examples/IntersectExample.java | 281 +++++++
.../mapreduce/examples/IntersectValidate.java | 257 +++++++
.../org/apache/tez/mapreduce/examples/Join.java | 176 +++++
.../tez/mapreduce/examples/MRRSleepJob.java | 757 +++++++++++++++++++
.../tez/mapreduce/examples/MapredWordCount.java | 163 ++++
.../mapreduce/examples/RandomTextWriter.java | 757 +++++++++++++++++++
.../tez/mapreduce/examples/RandomWriter.java | 298 ++++++++
.../tez/mapreduce/examples/SecondarySort.java | 248 ++++++
.../org/apache/tez/mapreduce/examples/Sort.java | 203 +++++
.../examples/TestOrderedWordCount.java | 479 ++++++++++++
.../tez/mapreduce/examples/UnionExample.java | 295 ++++++++
.../helpers/SplitsInClientOptionParser.java | 72 ++
.../processor/FilterByWordInputProcessor.java | 135 ++++
.../processor/FilterByWordOutputProcessor.java | 96 +++
.../mapreduce/examples/terasort/GenSort.java | 251 ++++++
.../mapreduce/examples/terasort/Random16.java | 374 +++++++++
.../examples/terasort/TeraChecksum.java | 103 +++
.../mapreduce/examples/terasort/TeraGen.java | 311 ++++++++
.../examples/terasort/TeraInputFormat.java | 353 +++++++++
.../examples/terasort/TeraOutputFormat.java | 112 +++
.../examples/terasort/TeraScheduler.java | 253 +++++++
.../mapreduce/examples/terasort/TeraSort.java | 335 ++++++++
.../examples/terasort/TeraValidate.java | 188 +++++
.../mapreduce/examples/terasort/Unsigned16.java | 297 ++++++++
.../examples/terasort/TestTeraSort.java | 94 +++
88 files changed, 9360 insertions(+), 9912 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index 14753b9..34149e7 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -22,9 +22,9 @@ Maven main modules:
- tez-runtime-library .........(Tez runtime library)
- tez-mapreduce ...............(Tez mapreduce)
- tez-dag .....................(Tez dag)
- - tez-mapreduce-examples ......(Tez mapreduce examples)
+ - tez-examples ................(Tez examples)
- tez-plugins .................(Tez plugins)
- - tez-tests ...................(Tez tests)
+ - tez-tests ...................(Tez tests and additional test examples)
- tez-dist ....................(Tez dist)
----------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 59057e7..3b3f951 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -64,6 +64,8 @@ INCOMPATIBLE CHANGES
TEZ-1320. Remove getApplicationId from DAGClient
TEZ-1427. Change remaining classes that are using byte[] to UserPayload
TEZ-1418. Provide Default value for TEZ_AM_LAUNCH_ENV and TEZ_TASK_LAUNCH
+ TEZ-1438. Annotate add java doc for tez-runtime-library and tez-mapreduce
+ TEZ-1055. Rename tez-mapreduce-examples to tez-examples
Release 0.4.0-incubating: 2014-04-05
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/INSTALL.txt
----------------------------------------------------------------------
diff --git a/INSTALL.txt b/INSTALL.txt
index f8385fd..cd29fce 100644
--- a/INSTALL.txt
+++ b/INSTALL.txt
@@ -40,28 +40,26 @@ $HADOOP_PREFIX/bin/hadoop jar hadoop-mapreduce-client-jobclient-2.2.0-tests.jar
This will use the TEZ DAG ApplicationMaster to run the MR job. This can be
verified by looking at the AM's logs from the YARN ResourceManager UI.
-7) There is a basic example of using an MRR job in the tez-mapreduce-examples.jar. Refer to TestOrderedWordCount.java
+7) There is a basic example of using an MRR job in the tez-examples.jar. Refer to OrderedWordCount.java
in the source code. To run this example:
-$HADOOP_PREFIX/bin/hadoop jar tez-mapreduce-examples.jar testorderedwordcount <input> <output>
+$HADOOP_PREFIX/bin/hadoop jar tez-examples.jar orderedwordcount <input> <output>
This will use the TEZ DAG ApplicationMaster to run the ordered word count job. This job is similar
to the word count example except that it also orders all words based on the frequency of
occurrence.
-There are multiple variations to run testorderedwordcount. You can use it to run multiple
+There are multiple variations of orderedwordcount. You can take a look at TestOrderedWordCount.java
+in tez-tests for these variations. You can use it to run multiple
DAGs serially on different inputs/outputs. These DAGs could be run separately as
different applications or serially within a single TEZ session.
-$HADOOP_PREFIX/bin/hadoop jar tez-mapreduce-examples.jar testorderedwordcount <input1> <output1> <input2> <output2> <input3> <output3> ...
+$HADOOP_PREFIX/bin/hadoop jar tez-tests.jar testorderedwordcount <input1> <output1> <input2> <output2> <input3> <output3> ...
The above will run multiple DAGs for each input-output pair. To use TEZ sessions,
set -DUSE_TEZ_SESSION=true
-$HADOOP_PREFIX/bin/hadoop jar tez-mapreduce-examples.jar testorderedwordcount -DUSE_TEZ_SESSION=true <input1> <output1> <input2> <output2>
-
-
-
+$HADOOP_PREFIX/bin/hadoop jar tez-tests.jar testorderedwordcount -DUSE_TEZ_SESSION=true <input1> <output1> <input2> <output2>
Alternate machanism to setup Tez to use Hadoop libraries from the cluster.
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/docs/src/site/apt/install_0_5_0.apt
----------------------------------------------------------------------
diff --git a/docs/src/site/apt/install_0_5_0.apt b/docs/src/site/apt/install_0_5_0.apt
index 90a94c2..0cabeaa 100644
--- a/docs/src/site/apt/install_0_5_0.apt
+++ b/docs/src/site/apt/install_0_5_0.apt
@@ -88,22 +88,22 @@ $HADOOP_PREFIX/bin/hadoop jar hadoop-mapreduce-client-jobclient-3.0.0-SNAPSHOT-t
This will use the TEZ DAG ApplicationMaster to run the MR job. This can be verified by looking at the AM's logs from the YARN ResourceManager UI.
- [[i]] There is a basic example of using an MRR job in the tez-mapreduce-examples.jar. Refer to OrderedWordCount.java
+ [[i]] There is a basic example of using an MRR job in the tez-examples.jar. Refer to OrderedWordCount.java
in the source code. To run this example:
+---+
-$HADOOP_PREFIX/bin/hadoop jar tez-mapreduce-examples.jar orderedwordcount <input> <output>
+$HADOOP_PREFIX/bin/hadoop jar tez-examples.jar orderedwordcount <input> <output>
+---+
This will use the TEZ DAG ApplicationMaster to run the ordered word count job. This job is similar to the word count example except that it also orders all words based on the frequency of occurrence.
- There are multiple variations to run orderedwordcount. You can use it to run multiple DAGs serially on different inputs/outputs. These DAGs could be run separately as different applications or serially within a single TEZ session.
+ Tez DAGs could be run separately as different applications or serially within a single TEZ session. There is a different variation of orderedwordcount in tez-tests that supports the use of Sessions and handling multiple input-output pairs. You can use it to run multiple DAGs serially on different inputs/outputs.
+---+
-$HADOOP_PREFIX/bin/hadoop jar tez-mapreduce-examples.jar orderedwordcount <input1> <output1> <input2> <output2> <input3> <output3> ...
+$HADOOP_PREFIX/bin/hadoop jar tez-tests.jar testorderedwordcount <input1> <output1> <input2> <output2> <input3> <output3> ...
+---+
@@ -113,6 +113,6 @@ $HADOOP_PREFIX/bin/hadoop jar tez-mapreduce-examples.jar orderedwordcount <input
+---+
-$HADOOP_PREFIX/bin/hadoop jar tez-mapreduce-examples.jar orderedwordcount -DUSE_TEZ_SESSION=true <input1> <output1> <input2> <output2>
+$HADOOP_PREFIX/bin/hadoop jar tez-tests.jar testorderedwordcount -DUSE_TEZ_SESSION=true <input1> <output1> <input2> <output2>
+---+
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b58ab96..cab8633 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,7 +126,7 @@
</dependency>
<dependency>
<groupId>org.apache.tez</groupId>
- <artifactId>tez-mapreduce-examples</artifactId>
+ <artifactId>tez-examples</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
@@ -603,7 +603,7 @@
<module>tez-runtime-library</module>
<module>tez-runtime-internals</module>
<module>tez-mapreduce</module>
- <module>tez-mapreduce-examples</module>
+ <module>tez-examples</module>
<module>tez-tests</module>
<module>tez-dag</module>
<module>tez-plugins</module>
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-dist/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dist/pom.xml b/tez-dist/pom.xml
index cf1858a..d7eaaa5 100644
--- a/tez-dist/pom.xml
+++ b/tez-dist/pom.xml
@@ -35,7 +35,7 @@
</dependency>
<dependency>
<groupId>org.apache.tez</groupId>
- <artifactId>tez-mapreduce-examples</artifactId>
+ <artifactId>tez-examples</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-dist/src/main/assembly/tez-dist-minimal.xml
----------------------------------------------------------------------
diff --git a/tez-dist/src/main/assembly/tez-dist-minimal.xml b/tez-dist/src/main/assembly/tez-dist-minimal.xml
index 8c4a58b..60aa50e 100644
--- a/tez-dist/src/main/assembly/tez-dist-minimal.xml
+++ b/tez-dist/src/main/assembly/tez-dist-minimal.xml
@@ -22,9 +22,6 @@
<moduleSets>
<moduleSet>
<useAllReactorProjects>true</useAllReactorProjects>
- <excludes>
- <exclude>org.apache.tez:tez-tests</exclude>
- </excludes>
<binaries>
<outputDirectory>/</outputDirectory>
<unpack>false</unpack>
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-dist/src/main/assembly/tez-dist.xml
----------------------------------------------------------------------
diff --git a/tez-dist/src/main/assembly/tez-dist.xml b/tez-dist/src/main/assembly/tez-dist.xml
index cb901ef..9b2aa04 100644
--- a/tez-dist/src/main/assembly/tez-dist.xml
+++ b/tez-dist/src/main/assembly/tez-dist.xml
@@ -22,9 +22,6 @@
<moduleSets>
<moduleSet>
<useAllReactorProjects>true</useAllReactorProjects>
- <excludes>
- <exclude>org.apache.tez:tez-tests</exclude>
- </excludes>
<binaries>
<outputDirectory>/</outputDirectory>
<unpack>false</unpack>
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-examples/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/tez-examples/findbugs-exclude.xml b/tez-examples/findbugs-exclude.xml
new file mode 100644
index 0000000..5b11308
--- /dev/null
+++ b/tez-examples/findbugs-exclude.xml
@@ -0,0 +1,16 @@
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<FindBugsFilter>
+
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-examples/pom.xml
----------------------------------------------------------------------
diff --git a/tez-examples/pom.xml b/tez-examples/pom.xml
new file mode 100644
index 0000000..04ee9a0
--- /dev/null
+++ b/tez-examples/pom.xml
@@ -0,0 +1,100 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez</artifactId>
+ <version>0.5.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>tez-examples</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-runtime-library</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-mapreduce</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <configuration>
+ <archive>
+ <manifest>
+ <mainClass>org.apache.tez.examples.ExampleDriver</mainClass>
+ </manifest>
+ </archive>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-examples/src/main/java/org/apache/tez/examples/ExampleDriver.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/ExampleDriver.java b/tez-examples/src/main/java/org/apache/tez/examples/ExampleDriver.java
new file mode 100644
index 0000000..c664692
--- /dev/null
+++ b/tez-examples/src/main/java/org/apache/tez/examples/ExampleDriver.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.examples;
+
+import java.io.IOException;
+import java.text.DecimalFormat;
+import java.util.EnumSet;
+import java.util.Set;
+
+import org.apache.hadoop.util.ProgramDriver;
+import org.apache.tez.common.counters.TezCounters;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.api.client.Progress;
+import org.apache.tez.dag.api.client.StatusGetOpts;
+import org.apache.tez.dag.api.client.VertexStatus;
+
+import sun.misc.Sort;
+
+/**
+ * A description of an example program based on its class and a
+ * human-readable description.
+ */
+public class ExampleDriver {
+
+ private static final DecimalFormat formatter = new DecimalFormat("###.##%");
+
+ public static void main(String argv[]){
+ int exitCode = -1;
+ ProgramDriver pgd = new ProgramDriver();
+ try {
+ pgd.addClass("wordcount", WordCount.class,
+ "A native Tez wordcount program that counts the words in the input files.");
+ pgd.addClass("orderedwordcount", OrderedWordCount.class,
+ "Word Count with words sorted on frequency");
+ pgd.addClass("simplesessionexample", SimpleSessionExample.class,
+ "Example to run multiple dags in a session");
+ exitCode = pgd.run(argv);
+ } catch(Throwable e){
+ e.printStackTrace();
+ }
+
+ System.exit(exitCode);
+ }
+
+ public static void printDAGStatus(DAGClient dagClient, String[] vertexNames)
+ throws IOException, TezException {
+ printDAGStatus(dagClient, vertexNames, false, false);
+ }
+
+ public static void printDAGStatus(DAGClient dagClient, String[] vertexNames,
+ boolean displayDAGCounters, boolean displayVertexCounters)
+ throws IOException, TezException {
+ Set<StatusGetOpts> opts = EnumSet.of(StatusGetOpts.GET_COUNTERS);
+ DAGStatus dagStatus = dagClient.getDAGStatus(
+ (displayDAGCounters ? opts : null));
+ Progress progress = dagStatus.getDAGProgress();
+ double vProgressFloat = 0.0f;
+ if (progress != null) {
+ System.out.println("");
+ System.out.println("DAG: State: "
+ + dagStatus.getState()
+ + " Progress: "
+ + (progress.getTotalTaskCount() < 0 ? formatter.format(0.0f) :
+ formatter.format((double)(progress.getSucceededTaskCount())
+ /progress.getTotalTaskCount())));
+ for (String vertexName : vertexNames) {
+ VertexStatus vStatus = dagClient.getVertexStatus(vertexName,
+ (displayVertexCounters ? opts : null));
+ if (vStatus == null) {
+ System.out.println("Could not retrieve status for vertex: "
+ + vertexName);
+ continue;
+ }
+ Progress vProgress = vStatus.getProgress();
+ if (vProgress != null) {
+ vProgressFloat = 0.0f;
+ if (vProgress.getTotalTaskCount() == 0) {
+ vProgressFloat = 1.0f;
+ } else if (vProgress.getTotalTaskCount() > 0) {
+ vProgressFloat = (double)vProgress.getSucceededTaskCount()
+ /vProgress.getTotalTaskCount();
+ }
+ System.out.println("VertexStatus:"
+ + " VertexName: "
+ + (vertexName.equals("ivertex1") ? "intermediate-reducer"
+ : vertexName)
+ + " Progress: " + formatter.format(vProgressFloat));
+ }
+ if (displayVertexCounters) {
+ TezCounters counters = vStatus.getVertexCounters();
+ if (counters != null) {
+ System.out.println("Vertex Counters for " + vertexName + ": "
+ + counters);
+ }
+ }
+ }
+ }
+ if (displayDAGCounters) {
+ TezCounters counters = dagStatus.getDAGCounters();
+ if (counters != null) {
+ System.out.println("DAG Counters: " + counters);
+ }
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java b/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
new file mode 100644
index 0000000..8ba8179
--- /dev/null
+++ b/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.examples;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.tez.client.TezClient;
+import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.DataSinkDescriptor;
+import org.apache.tez.dag.api.DataSourceDescriptor;
+import org.apache.tez.dag.api.Edge;
+import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.examples.WordCount.TokenProcessor;
+import org.apache.tez.mapreduce.input.MRInput;
+import org.apache.tez.mapreduce.output.MROutput;
+import org.apache.tez.mapreduce.processor.SimpleMRProcessor;
+import org.apache.tez.runtime.api.ProcessorContext;
+import org.apache.tez.runtime.library.api.KeyValueWriter;
+import org.apache.tez.runtime.library.api.KeyValuesReader;
+import org.apache.tez.runtime.library.conf.OrderedPartitionedKVEdgeConfigurer;
+import org.apache.tez.runtime.library.partitioner.HashPartitioner;
+import org.apache.tez.runtime.library.processor.SimpleProcessor;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Simple example that extends the WordCount example to show a chain of processing.
+ * The example extends WordCount by sorting the words by their count.
+ */
+public class OrderedWordCount extends Configured implements Tool {
+
+ /*
+ * SumProcessor similar to WordCount except that it writes the count as key and the
+ * word as value. This is because we can and ordered partitioned key value edge to group the
+ * words with the same count (as key) and order the counts.
+ */
+ public static class SumProcessor extends SimpleProcessor {
+ public SumProcessor(ProcessorContext context) {
+ super(context);
+ }
+
+ @Override
+ public void run() throws Exception {
+ Preconditions.checkArgument(getInputs().size() == 1);
+ Preconditions.checkArgument(getOutputs().size() == 1);
+ KeyValueWriter kvWriter = (KeyValueWriter) getOutputs().values().iterator().next().getWriter();
+ KeyValuesReader kvReader = (KeyValuesReader) getInputs().values().iterator().next().getReader();
+ while (kvReader.next()) {
+ Text word = (Text) kvReader.getCurrentKey();
+ int sum = 0;
+ for (Object value : kvReader.getCurrentValues()) {
+ sum += ((IntWritable) value).get();
+ }
+ // write the sum as the key and the word as the value
+ kvWriter.write(new IntWritable(sum), word);
+ }
+ }
+ }
+
+ /**
+ * No-op sorter processor. It does not need to apply any logic since the ordered partitioned edge
+ * ensures that we get the data sorted and grouped by the the sum key.
+ */
+ public static class NoOpSorter extends SimpleMRProcessor {
+
+ public NoOpSorter(ProcessorContext context) {
+ super(context);
+ }
+
+ @Override
+ public void run() throws Exception {
+ Preconditions.checkArgument(getInputs().size() == 1);
+ Preconditions.checkArgument(getOutputs().size() == 1);
+ KeyValueWriter kvWriter = (KeyValueWriter) getOutputs().values().iterator().next().getWriter();
+ KeyValuesReader kvReader = (KeyValuesReader) getInputs().values().iterator().next().getReader();
+ while (kvReader.next()) {
+ Object sum = kvReader.getCurrentKey();
+ for (Object word : kvReader.getCurrentValues()) {
+ kvWriter.write(word, sum);
+ }
+ }
+ // deriving from SimpleMRProcessor takes care of committing the output
+ }
+ }
+
+ public static DAG createDAG(TezConfiguration tezConf, String inputPath, String outputPath,
+ int numPartitions, String dagName) throws IOException {
+
+ DataSourceDescriptor dataSource = MRInput.createConfigurer(new Configuration(tezConf),
+ TextInputFormat.class, inputPath).create();
+
+ DataSinkDescriptor dataSink = MROutput.createConfigurer(new Configuration(tezConf),
+ TextOutputFormat.class, outputPath).create();
+
+ Vertex tokenizerVertex = new Vertex("Tokenizer", new ProcessorDescriptor(
+ TokenProcessor.class.getName()));
+ tokenizerVertex.addDataSource("MRInput", dataSource);
+
+ // Use Text key and IntWritable value to bring counts for each word in the same partition
+ OrderedPartitionedKVEdgeConfigurer summationEdgeConf = OrderedPartitionedKVEdgeConfigurer
+ .newBuilder(Text.class.getName(), IntWritable.class.getName(),
+ HashPartitioner.class.getName()).build();
+
+ // This vertex will be reading intermediate data via an input edge and writing intermediate data
+ // via an output edge.
+ Vertex summationVertex = new Vertex("Summation", new ProcessorDescriptor(
+ SumProcessor.class.getName()), numPartitions);
+
+ // Use IntWritable key and Text value to bring all words with the same count in the same
+ // partition. The data will be ordered by count and words grouped by count.
+ OrderedPartitionedKVEdgeConfigurer sorterEdgeConf = OrderedPartitionedKVEdgeConfigurer
+ .newBuilder(IntWritable.class.getName(), Text.class.getName(),
+ HashPartitioner.class.getName()).build();
+
+ // Use 1 task to bring all the data in one place for global sorted order. Essentially the number
+ // of partitions is 1. So the NoOpSorter can be used to produce the globally ordered output
+ Vertex sorterVertex = new Vertex("Sorter", new ProcessorDescriptor(
+ NoOpSorter.class.getName()), 1);
+ sorterVertex.addDataSink("MROutput", dataSink);
+
+ // No need to add jar containing this class as assumed to be part of the tez jars.
+
+ DAG dag = new DAG(dagName);
+ dag.addVertex(tokenizerVertex)
+ .addVertex(summationVertex)
+ .addVertex(sorterVertex)
+ .addEdge(
+ new Edge(tokenizerVertex, summationVertex, summationEdgeConf.createDefaultEdgeProperty()))
+ .addEdge(
+ new Edge(summationVertex, sorterVertex, sorterEdgeConf.createDefaultEdgeProperty()));
+ return dag;
+ }
+
+ private static void printUsage() {
+ System.err.println("Usage: " + " orderedwordcount in out [numPartitions]");
+ ToolRunner.printGenericCommandUsage(System.err);
+ }
+
+ public boolean run(String inputPath, String outputPath, Configuration conf,
+ int numPartitions) throws Exception {
+ System.out.println("Running OrderedWordCount");
+ TezConfiguration tezConf;
+ if (conf != null) {
+ tezConf = new TezConfiguration(conf);
+ } else {
+ tezConf = new TezConfiguration();
+ }
+
+ TezClient tezClient = new TezClient("OrderedWordCount", tezConf);
+ tezClient.start();
+
+ try {
+ DAG dag = createDAG(tezConf, inputPath, outputPath, numPartitions, "OrderedWordCount");
+
+ tezClient.waitTillReady();
+ DAGClient dagClient = tezClient.submitDAG(dag);
+
+ // monitoring
+ DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(null);
+ if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
+ System.out.println("OrderedWordCount failed with diagnostics: " + dagStatus.getDiagnostics());
+ return false;
+ }
+ return true;
+ } finally {
+ tezClient.stop();
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Configuration conf = getConf();
+ String [] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+
+ if (otherArgs.length < 2 || otherArgs.length > 3) {
+ printUsage();
+ return 2;
+ }
+ OrderedWordCount job = new OrderedWordCount();
+ if (job.run(otherArgs[0], otherArgs[1], conf,
+ (otherArgs.length == 3 ? Integer.parseInt(otherArgs[2]) : 1))) {
+ return 0;
+ }
+ return 1;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(), new OrderedWordCount(), args);
+ System.exit(res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-examples/src/main/java/org/apache/tez/examples/SimpleSessionExample.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/SimpleSessionExample.java b/tez-examples/src/main/java/org/apache/tez/examples/SimpleSessionExample.java
new file mode 100644
index 0000000..1d51e9e
--- /dev/null
+++ b/tez-examples/src/main/java/org/apache/tez/examples/SimpleSessionExample.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.examples;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.tez.client.PreWarmVertex;
+import org.apache.tez.client.TezClient;
+import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+
+/**
+ * Simple example that shows how Tez session mode can be used to run multiple DAGs in the same
+ * session for efficiency and performance. Tez session mode enables the execution environment to
+ * hold onto execution resources so that they can be reused to across multiple DAGs. Typical use
+ * cases for this would be drill-down queries from a user shell or maintaining a pool of resources
+ * to provide low latency execution of DAGs for an application.
+ * In this example we will be submitting multiple OrderedWordCount DAGs on different inputs to the
+ * same session.
+ */
+public class SimpleSessionExample extends Configured implements Tool {
+
+ private static final String enablePrewarmConfig = "simplesessionexample.prewarm";
+
+ public boolean run(String[] inputPaths, String[] outputPaths, Configuration conf,
+ int numPartitions) throws Exception {
+ TezConfiguration tezConf;
+ if (conf != null) {
+ tezConf = new TezConfiguration(conf);
+ } else {
+ tezConf = new TezConfiguration();
+ }
+
+ // start TezClient in session mode. The same code run in session mode or non-session mode. The
+ // mode can be changed via configuration. However if the application wants to run exclusively in
+ // session mode then it can do so in code directly using the appropriate constructor
+
+ // tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true); // via config OR via code
+ TezClient tezClient = new TezClient("SimpleSessionExample", tezConf, true);
+ tezClient.start();
+
+ // Session pre-warming allows the user to hide initial startup, resource acquisition latency etc.
+ // by pre-allocating execution resources in the Tez session. They can run initialization logic
+ // in these pre-allocated resources (containers) to pre-warm the containers.
+ // In between DAG executions, the session can hold on to a minimum number of containers.
+ // Ideally, this would be enough to provide desired balance of efficiency for the application
+ // and sharing of resources with other applications. Typically, the number of containers to be
+ // pre-warmed equals the number of containers to be held between DAGs.
+ if (tezConf.getBoolean(enablePrewarmConfig, false)) {
+ // the above parameter is not a Tez parameter. Its only for this example.
+ // In this example we are pre-warming enough containers to run all the sum tasks in parallel.
+ // This means pre-warming numPartitions number of containers.
+ // We are making the pre-warm and held containers to be the same and using the helper API to
+ // set up pre-warming. They can be made different and also custom initialization logic can be
+ // specified using other API's. We know that the OrderedWordCount dag uses default files and
+ // resources. Otherwise we would have to specify matching parameters in the preWarm API too.
+ tezConf.setInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, numPartitions);
+ tezClient.preWarm(PreWarmVertex.createConfigurer(tezConf).create());
+ }
+
+ // the remaining code is the same as submitting any DAG.
+ try {
+ for (int i=0; i<inputPaths.length; ++i) {
+ DAG dag = OrderedWordCount.createDAG(tezConf, inputPaths[i], outputPaths[i], numPartitions,
+ ("DAG-Iteration-" + i)); // the names of the DAGs must be unique in a session
+
+ tezClient.waitTillReady();
+ System.out.println("Running dag number " + i);
+ DAGClient dagClient = tezClient.submitDAG(dag);
+
+ // wait to finish
+ DAGStatus dagStatus = dagClient.waitForCompletion();
+ if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
+ System.out.println("Iteration " + i + " failed with diagnostics: "
+ + dagStatus.getDiagnostics());
+ return false;
+ }
+ }
+ return true;
+ } finally {
+ tezClient.stop();
+ }
+ }
+
+ private static void printUsage() {
+ System.err.println("Usage: " + " simplesessionexample <in1,in2> <out1, out2> [numPartitions]");
+ ToolRunner.printGenericCommandUsage(System.err);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ System.out.println("Running SimpleSessionExample");
+ Configuration conf = getConf();
+ String [] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+
+ if (otherArgs.length < 2 || otherArgs.length > 3) {
+ printUsage();
+ return 2;
+ }
+
+ String[] inputPaths = otherArgs[0].split(",");
+ String[] outputPaths = otherArgs[1].split(",");
+ if (inputPaths.length != outputPaths.length) {
+ System.err.println("Inputs and outputs must be equal in number");
+ return 3;
+ }
+
+ SimpleSessionExample job = new SimpleSessionExample();
+ if (job.run(inputPaths, outputPaths, conf,
+ (otherArgs.length == 3 ? Integer.parseInt(otherArgs[2]) : 1))) {
+ return 0;
+ }
+ return 1;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(), new SimpleSessionExample(), args);
+ System.exit(res);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java b/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
new file mode 100644
index 0000000..6e49dbc
--- /dev/null
+++ b/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
@@ -0,0 +1,254 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.examples;
+
+import java.io.IOException;
+import java.util.StringTokenizer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.tez.client.TezClient;
+import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.DataSinkDescriptor;
+import org.apache.tez.dag.api.DataSourceDescriptor;
+import org.apache.tez.dag.api.Edge;
+import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.mapreduce.input.MRInput;
+import org.apache.tez.mapreduce.output.MROutput;
+import org.apache.tez.mapreduce.processor.SimpleMRProcessor;
+import org.apache.tez.runtime.api.ProcessorContext;
+import org.apache.tez.runtime.library.api.KeyValueReader;
+import org.apache.tez.runtime.library.api.KeyValueWriter;
+import org.apache.tez.runtime.library.api.KeyValuesReader;
+import org.apache.tez.runtime.library.conf.OrderedPartitionedKVEdgeConfigurer;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.tez.runtime.library.partitioner.HashPartitioner;
+import org.apache.tez.runtime.library.processor.SimpleProcessor;
+
+/**
+ * Simple example to perform WordCount using Tez API's. WordCount is the
+ * HelloWorld program of distributed data processing and counts the number
+ * of occurrences of a word in a distributed text data set.
+ */
+public class WordCount extends Configured implements Tool {
+
+ /*
+ * Example code to write a processor in Tez.
+ * Processors typically apply the main application logic to the data.
+ * TokenProcessor tokenizes the input data.
+ * It uses an input that provide a Key-Value reader and writes
+ * output to a Key-Value writer. The processor inherits from SimpleProcessor
+ * since it does not need to handle any advanced constructs for Processors.
+ */
+ public static class TokenProcessor extends SimpleProcessor {
+ IntWritable one = new IntWritable(1);
+ Text word = new Text();
+
+ public TokenProcessor(ProcessorContext context) {
+ super(context);
+ }
+
+ @Override
+ public void run() throws Exception {
+ Preconditions.checkArgument(getInputs().size() == 1);
+ Preconditions.checkArgument(getOutputs().size() == 1);
+ // the recommended approach is to cast the reader/writer to a specific type instead
+ // of casting the input/output. This allows the actual input/output type to be replaced
+ // without affecting the semantic guarantees of the data type that are represented by
+ // the reader and writer.
+ KeyValueReader kvReader = (KeyValueReader) getInputs().values().iterator().next().getReader();
+ KeyValueWriter kvWriter = (KeyValueWriter) getOutputs().values().iterator().next().getWriter();
+ while (kvReader.next()) {
+ StringTokenizer itr = new StringTokenizer(kvReader.getCurrentValue().toString());
+ while (itr.hasMoreTokens()) {
+ word.set(itr.nextToken());
+ // Count 1 every time a word is observed. Word is the key a 1 is the value
+ kvWriter.write(word, one);
+ }
+ }
+ }
+
+ }
+
+ /*
+ * Example code to write a processor that commits final output to a data sink
+ * The SumProcessor aggregates the sum of individual word counts generated by
+ * the TokenProcessor.
+ * The SumProcessor is connected to a DataSink. In this case, its an Output that
+ * writes the data via an OutputFormat to a data sink (typically HDFS). Thats why
+ * it derives from SimpleMRProcessor that takes care of handling the necessary
+ * output commit operations that makes the final output available for consumers.
+ */
+ public static class SumProcessor extends SimpleMRProcessor {
+ public SumProcessor(ProcessorContext context) {
+ super(context);
+ }
+
+ @Override
+ public void run() throws Exception {
+ Preconditions.checkArgument(getInputs().size() == 1);
+ Preconditions.checkArgument(getOutputs().size() == 1);
+ KeyValueWriter kvWriter = (KeyValueWriter) getOutputs().values().iterator().next().getWriter();
+ // The KeyValues reader provides all values for a given key. The aggregation of values per key
+ // is done by the LogicalInput. Since the key is the word and the values are its counts in
+ // the different TokenProcessors, summing all values per key provides the sum for that word.
+ KeyValuesReader kvReader = (KeyValuesReader) getInputs().values().iterator().next().getReader();
+ while (kvReader.next()) {
+ Text word = (Text) kvReader.getCurrentKey();
+ int sum = 0;
+ for (Object value : kvReader.getCurrentValues()) {
+ sum += ((IntWritable) value).get();
+ }
+ kvWriter.write(word, new IntWritable(sum));
+ }
+ // deriving from SimpleMRProcessor takes care of committing the output
+ // It automatically invokes the commit logic for the OutputFormat if necessary.
+ }
+ }
+
+ private DAG createDAG(TezConfiguration tezConf, String inputPath, String outputPath,
+ int numPartitions) throws IOException {
+
+ // Create the descriptor that describes the input data to Tez. Using MRInput to read text
+ // data from the given input path. The TextInputFormat is used to read the text data.
+ DataSourceDescriptor dataSource = MRInput.createConfigurer(new Configuration(tezConf),
+ TextInputFormat.class, inputPath).create();
+
+ // Create a descriptor that describes the output data to Tez. Using MROoutput to write text
+ // data to the given output path. The TextOutputFormat is used to write the text data.
+ DataSinkDescriptor dataSink = MROutput.createConfigurer(new Configuration(tezConf),
+ TextOutputFormat.class, outputPath).create();
+
+ // Create a vertex that reads the data from the data source and tokenizes it using the
+ // TokenProcessor. The number of tasks that will do the work for this vertex will be decided
+ // using the information provided by the data source descriptor.
+ Vertex tokenizerVertex = new Vertex("Tokenizer", new ProcessorDescriptor(
+ TokenProcessor.class.getName())).addDataSource("Input", dataSource);
+
+ // Create the edge that represents the movement and semantics of data between the producer
+ // Tokenizer vertex and the consumer Summation vertex. In order to perform the summation in
+ // parallel the tokenized data will be partitioned by word such that a given word goes to the
+ // same partition. The counts for the words should be grouped together per word. To achieve this
+ // we can use an edge that contains an input/output pair that handles partitioning and grouping
+ // of key value data. We use the helper OrderedPartitionedKVEdgeConfigurer to create such an
+ // edge. Internally, it sets up matching Tez inputs and outputs that can perform this logic.
+ // We specify the key, value and partitioner type. Here the key type is Text (for word), the
+ // value type is IntWritable (for count) and we using a hash based partitioner. This is a helper
+ // object. The edge can be configured by configuring the input, output etc individually without
+ // using this helper.
+ OrderedPartitionedKVEdgeConfigurer edgeConf = OrderedPartitionedKVEdgeConfigurer
+ .newBuilder(Text.class.getName(), IntWritable.class.getName(),
+ HashPartitioner.class.getName()).build();
+
+ // Create a vertex that reads the tokenized data and calculates the sum using the SumProcessor.
+ // The number of tasks that do the work of this vertex depends on the number of partitions used
+ // to distribute the sum processing. In this case, its been made configurable via the
+ // numPartitions parameter.
+ Vertex summationVertex = new Vertex("Summation",
+ new ProcessorDescriptor(SumProcessor.class.getName()), numPartitions)
+ .addDataSink("Output", dataSink);
+
+ // No need to add jar containing this class as assumed to be part of the Tez jars. Otherwise
+ // we would have to add the jars for this code as local files to the vertices.
+
+ // Create DAG and add the vertices. Connect the producer and consumer vertices via the edge
+ DAG dag = new DAG("WordCount");
+ dag.addVertex(tokenizerVertex)
+ .addVertex(summationVertex)
+ .addEdge(
+ new Edge(tokenizerVertex, summationVertex, edgeConf.createDefaultEdgeProperty()));
+ return dag;
+ }
+
+ private static void printUsage() {
+ System.err.println("Usage: " + " wordcount in out [numPartitions]");
+ ToolRunner.printGenericCommandUsage(System.err);
+ }
+
+ public boolean run(String inputPath, String outputPath, Configuration conf,
+ int numPartitions) throws Exception {
+ System.out.println("Running WordCount");
+ TezConfiguration tezConf;
+ if (conf != null) {
+ tezConf = new TezConfiguration(conf);
+ } else {
+ tezConf = new TezConfiguration();
+ }
+
+ // Create the TezClient to submit the DAG. Pass the tezConf that has all necessary global and
+ // dag specific configurations
+ TezClient tezClient = new TezClient("WordCount", tezConf);
+ // TezClient must be started before it can be used
+ tezClient.start();
+
+ try {
+ DAG dag = createDAG(tezConf, inputPath, outputPath, numPartitions);
+
+ // check that the execution environment is ready
+ tezClient.waitTillReady();
+ // submit the dag and receive a dag client to monitor the progress
+ DAGClient dagClient = tezClient.submitDAG(dag);
+
+ // monitor the progress and wait for completion. This method blocks until the dag is done.
+ DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(null);
+ // check success or failure and print diagnostics
+ if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
+ System.out.println("WordCount failed with diagnostics: " + dagStatus.getDiagnostics());
+ return false;
+ }
+ return true;
+ } finally {
+ // stop the client to perform cleanup
+ tezClient.stop();
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Configuration conf = getConf();
+ String [] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+ if (otherArgs.length < 2 || otherArgs.length > 3) {
+ printUsage();
+ return 2;
+ }
+ WordCount job = new WordCount();
+ if (job.run(otherArgs[0], otherArgs[1], conf,
+ (otherArgs.length == 3 ? Integer.parseInt(otherArgs[2]) : 1))) {
+ return 0;
+ }
+ return 1;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(), new WordCount(), args);
+ System.exit(res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/findbugs-exclude.xml b/tez-mapreduce-examples/findbugs-exclude.xml
deleted file mode 100644
index 5b11308..0000000
--- a/tez-mapreduce-examples/findbugs-exclude.xml
+++ /dev/null
@@ -1,16 +0,0 @@
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-<FindBugsFilter>
-
-</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/pom.xml
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/pom.xml b/tez-mapreduce-examples/pom.xml
deleted file mode 100644
index 80245bc..0000000
--- a/tez-mapreduce-examples/pom.xml
+++ /dev/null
@@ -1,119 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
- http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.tez</groupId>
- <artifactId>tez</artifactId>
- <version>0.5.0-SNAPSHOT</version>
- </parent>
-
- <artifactId>tez-mapreduce-examples</artifactId>
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.tez</groupId>
- <artifactId>tez-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.tez</groupId>
- <artifactId>tez-common</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.tez</groupId>
- <artifactId>tez-runtime-library</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.tez</groupId>
- <artifactId>tez-mapreduce</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-common</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-common</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <scope>runtime</scope>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifest>
- <mainClass>org.apache.tez.mapreduce.examples.ExampleDriver</mainClass>
- </manifest>
- </archive>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- </plugin>
- </plugins>
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
deleted file mode 100644
index 8ba8179..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.examples;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.tez.client.TezClient;
-import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.DataSinkDescriptor;
-import org.apache.tez.dag.api.DataSourceDescriptor;
-import org.apache.tez.dag.api.Edge;
-import org.apache.tez.dag.api.ProcessorDescriptor;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.Vertex;
-import org.apache.tez.dag.api.client.DAGClient;
-import org.apache.tez.dag.api.client.DAGStatus;
-import org.apache.tez.examples.WordCount.TokenProcessor;
-import org.apache.tez.mapreduce.input.MRInput;
-import org.apache.tez.mapreduce.output.MROutput;
-import org.apache.tez.mapreduce.processor.SimpleMRProcessor;
-import org.apache.tez.runtime.api.ProcessorContext;
-import org.apache.tez.runtime.library.api.KeyValueWriter;
-import org.apache.tez.runtime.library.api.KeyValuesReader;
-import org.apache.tez.runtime.library.conf.OrderedPartitionedKVEdgeConfigurer;
-import org.apache.tez.runtime.library.partitioner.HashPartitioner;
-import org.apache.tez.runtime.library.processor.SimpleProcessor;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Simple example that extends the WordCount example to show a chain of processing.
- * The example extends WordCount by sorting the words by their count.
- */
-public class OrderedWordCount extends Configured implements Tool {
-
- /*
- * SumProcessor similar to WordCount except that it writes the count as key and the
- * word as value. This is because we can and ordered partitioned key value edge to group the
- * words with the same count (as key) and order the counts.
- */
- public static class SumProcessor extends SimpleProcessor {
- public SumProcessor(ProcessorContext context) {
- super(context);
- }
-
- @Override
- public void run() throws Exception {
- Preconditions.checkArgument(getInputs().size() == 1);
- Preconditions.checkArgument(getOutputs().size() == 1);
- KeyValueWriter kvWriter = (KeyValueWriter) getOutputs().values().iterator().next().getWriter();
- KeyValuesReader kvReader = (KeyValuesReader) getInputs().values().iterator().next().getReader();
- while (kvReader.next()) {
- Text word = (Text) kvReader.getCurrentKey();
- int sum = 0;
- for (Object value : kvReader.getCurrentValues()) {
- sum += ((IntWritable) value).get();
- }
- // write the sum as the key and the word as the value
- kvWriter.write(new IntWritable(sum), word);
- }
- }
- }
-
- /**
- * No-op sorter processor. It does not need to apply any logic since the ordered partitioned edge
- * ensures that we get the data sorted and grouped by the the sum key.
- */
- public static class NoOpSorter extends SimpleMRProcessor {
-
- public NoOpSorter(ProcessorContext context) {
- super(context);
- }
-
- @Override
- public void run() throws Exception {
- Preconditions.checkArgument(getInputs().size() == 1);
- Preconditions.checkArgument(getOutputs().size() == 1);
- KeyValueWriter kvWriter = (KeyValueWriter) getOutputs().values().iterator().next().getWriter();
- KeyValuesReader kvReader = (KeyValuesReader) getInputs().values().iterator().next().getReader();
- while (kvReader.next()) {
- Object sum = kvReader.getCurrentKey();
- for (Object word : kvReader.getCurrentValues()) {
- kvWriter.write(word, sum);
- }
- }
- // deriving from SimpleMRProcessor takes care of committing the output
- }
- }
-
- public static DAG createDAG(TezConfiguration tezConf, String inputPath, String outputPath,
- int numPartitions, String dagName) throws IOException {
-
- DataSourceDescriptor dataSource = MRInput.createConfigurer(new Configuration(tezConf),
- TextInputFormat.class, inputPath).create();
-
- DataSinkDescriptor dataSink = MROutput.createConfigurer(new Configuration(tezConf),
- TextOutputFormat.class, outputPath).create();
-
- Vertex tokenizerVertex = new Vertex("Tokenizer", new ProcessorDescriptor(
- TokenProcessor.class.getName()));
- tokenizerVertex.addDataSource("MRInput", dataSource);
-
- // Use Text key and IntWritable value to bring counts for each word in the same partition
- OrderedPartitionedKVEdgeConfigurer summationEdgeConf = OrderedPartitionedKVEdgeConfigurer
- .newBuilder(Text.class.getName(), IntWritable.class.getName(),
- HashPartitioner.class.getName()).build();
-
- // This vertex will be reading intermediate data via an input edge and writing intermediate data
- // via an output edge.
- Vertex summationVertex = new Vertex("Summation", new ProcessorDescriptor(
- SumProcessor.class.getName()), numPartitions);
-
- // Use IntWritable key and Text value to bring all words with the same count in the same
- // partition. The data will be ordered by count and words grouped by count.
- OrderedPartitionedKVEdgeConfigurer sorterEdgeConf = OrderedPartitionedKVEdgeConfigurer
- .newBuilder(IntWritable.class.getName(), Text.class.getName(),
- HashPartitioner.class.getName()).build();
-
- // Use 1 task to bring all the data in one place for global sorted order. Essentially the number
- // of partitions is 1. So the NoOpSorter can be used to produce the globally ordered output
- Vertex sorterVertex = new Vertex("Sorter", new ProcessorDescriptor(
- NoOpSorter.class.getName()), 1);
- sorterVertex.addDataSink("MROutput", dataSink);
-
- // No need to add jar containing this class as assumed to be part of the tez jars.
-
- DAG dag = new DAG(dagName);
- dag.addVertex(tokenizerVertex)
- .addVertex(summationVertex)
- .addVertex(sorterVertex)
- .addEdge(
- new Edge(tokenizerVertex, summationVertex, summationEdgeConf.createDefaultEdgeProperty()))
- .addEdge(
- new Edge(summationVertex, sorterVertex, sorterEdgeConf.createDefaultEdgeProperty()));
- return dag;
- }
-
- private static void printUsage() {
- System.err.println("Usage: " + " orderedwordcount in out [numPartitions]");
- ToolRunner.printGenericCommandUsage(System.err);
- }
-
- public boolean run(String inputPath, String outputPath, Configuration conf,
- int numPartitions) throws Exception {
- System.out.println("Running OrderedWordCount");
- TezConfiguration tezConf;
- if (conf != null) {
- tezConf = new TezConfiguration(conf);
- } else {
- tezConf = new TezConfiguration();
- }
-
- TezClient tezClient = new TezClient("OrderedWordCount", tezConf);
- tezClient.start();
-
- try {
- DAG dag = createDAG(tezConf, inputPath, outputPath, numPartitions, "OrderedWordCount");
-
- tezClient.waitTillReady();
- DAGClient dagClient = tezClient.submitDAG(dag);
-
- // monitoring
- DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(null);
- if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
- System.out.println("OrderedWordCount failed with diagnostics: " + dagStatus.getDiagnostics());
- return false;
- }
- return true;
- } finally {
- tezClient.stop();
- }
- }
-
- @Override
- public int run(String[] args) throws Exception {
- Configuration conf = getConf();
- String [] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
-
- if (otherArgs.length < 2 || otherArgs.length > 3) {
- printUsage();
- return 2;
- }
- OrderedWordCount job = new OrderedWordCount();
- if (job.run(otherArgs[0], otherArgs[1], conf,
- (otherArgs.length == 3 ? Integer.parseInt(otherArgs[2]) : 1))) {
- return 0;
- }
- return 1;
- }
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new OrderedWordCount(), args);
- System.exit(res);
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/examples/SimpleSessionExample.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/examples/SimpleSessionExample.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/examples/SimpleSessionExample.java
deleted file mode 100644
index 1d51e9e..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/examples/SimpleSessionExample.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.examples;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.tez.client.PreWarmVertex;
-import org.apache.tez.client.TezClient;
-import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.client.DAGClient;
-import org.apache.tez.dag.api.client.DAGStatus;
-
-/**
- * Simple example that shows how Tez session mode can be used to run multiple DAGs in the same
- * session for efficiency and performance. Tez session mode enables the execution environment to
- * hold onto execution resources so that they can be reused to across multiple DAGs. Typical use
- * cases for this would be drill-down queries from a user shell or maintaining a pool of resources
- * to provide low latency execution of DAGs for an application.
- * In this example we will be submitting multiple OrderedWordCount DAGs on different inputs to the
- * same session.
- */
-public class SimpleSessionExample extends Configured implements Tool {
-
- private static final String enablePrewarmConfig = "simplesessionexample.prewarm";
-
- public boolean run(String[] inputPaths, String[] outputPaths, Configuration conf,
- int numPartitions) throws Exception {
- TezConfiguration tezConf;
- if (conf != null) {
- tezConf = new TezConfiguration(conf);
- } else {
- tezConf = new TezConfiguration();
- }
-
- // start TezClient in session mode. The same code run in session mode or non-session mode. The
- // mode can be changed via configuration. However if the application wants to run exclusively in
- // session mode then it can do so in code directly using the appropriate constructor
-
- // tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true); // via config OR via code
- TezClient tezClient = new TezClient("SimpleSessionExample", tezConf, true);
- tezClient.start();
-
- // Session pre-warming allows the user to hide initial startup, resource acquisition latency etc.
- // by pre-allocating execution resources in the Tez session. They can run initialization logic
- // in these pre-allocated resources (containers) to pre-warm the containers.
- // In between DAG executions, the session can hold on to a minimum number of containers.
- // Ideally, this would be enough to provide desired balance of efficiency for the application
- // and sharing of resources with other applications. Typically, the number of containers to be
- // pre-warmed equals the number of containers to be held between DAGs.
- if (tezConf.getBoolean(enablePrewarmConfig, false)) {
- // the above parameter is not a Tez parameter. Its only for this example.
- // In this example we are pre-warming enough containers to run all the sum tasks in parallel.
- // This means pre-warming numPartitions number of containers.
- // We are making the pre-warm and held containers to be the same and using the helper API to
- // set up pre-warming. They can be made different and also custom initialization logic can be
- // specified using other API's. We know that the OrderedWordCount dag uses default files and
- // resources. Otherwise we would have to specify matching parameters in the preWarm API too.
- tezConf.setInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, numPartitions);
- tezClient.preWarm(PreWarmVertex.createConfigurer(tezConf).create());
- }
-
- // the remaining code is the same as submitting any DAG.
- try {
- for (int i=0; i<inputPaths.length; ++i) {
- DAG dag = OrderedWordCount.createDAG(tezConf, inputPaths[i], outputPaths[i], numPartitions,
- ("DAG-Iteration-" + i)); // the names of the DAGs must be unique in a session
-
- tezClient.waitTillReady();
- System.out.println("Running dag number " + i);
- DAGClient dagClient = tezClient.submitDAG(dag);
-
- // wait to finish
- DAGStatus dagStatus = dagClient.waitForCompletion();
- if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
- System.out.println("Iteration " + i + " failed with diagnostics: "
- + dagStatus.getDiagnostics());
- return false;
- }
- }
- return true;
- } finally {
- tezClient.stop();
- }
- }
-
- private static void printUsage() {
- System.err.println("Usage: " + " simplesessionexample <in1,in2> <out1, out2> [numPartitions]");
- ToolRunner.printGenericCommandUsage(System.err);
- }
-
- @Override
- public int run(String[] args) throws Exception {
- System.out.println("Running SimpleSessionExample");
- Configuration conf = getConf();
- String [] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
-
- if (otherArgs.length < 2 || otherArgs.length > 3) {
- printUsage();
- return 2;
- }
-
- String[] inputPaths = otherArgs[0].split(",");
- String[] outputPaths = otherArgs[1].split(",");
- if (inputPaths.length != outputPaths.length) {
- System.err.println("Inputs and outputs must be equal in number");
- return 3;
- }
-
- SimpleSessionExample job = new SimpleSessionExample();
- if (job.run(inputPaths, outputPaths, conf,
- (otherArgs.length == 3 ? Integer.parseInt(otherArgs[2]) : 1))) {
- return 0;
- }
- return 1;
- }
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new SimpleSessionExample(), args);
- System.exit(res);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/examples/WordCount.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/examples/WordCount.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/examples/WordCount.java
deleted file mode 100644
index 6e49dbc..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/examples/WordCount.java
+++ /dev/null
@@ -1,254 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tez.examples;
-
-import java.io.IOException;
-import java.util.StringTokenizer;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.tez.client.TezClient;
-import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.DataSinkDescriptor;
-import org.apache.tez.dag.api.DataSourceDescriptor;
-import org.apache.tez.dag.api.Edge;
-import org.apache.tez.dag.api.ProcessorDescriptor;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.Vertex;
-import org.apache.tez.dag.api.client.DAGClient;
-import org.apache.tez.dag.api.client.DAGStatus;
-import org.apache.tez.mapreduce.input.MRInput;
-import org.apache.tez.mapreduce.output.MROutput;
-import org.apache.tez.mapreduce.processor.SimpleMRProcessor;
-import org.apache.tez.runtime.api.ProcessorContext;
-import org.apache.tez.runtime.library.api.KeyValueReader;
-import org.apache.tez.runtime.library.api.KeyValueWriter;
-import org.apache.tez.runtime.library.api.KeyValuesReader;
-import org.apache.tez.runtime.library.conf.OrderedPartitionedKVEdgeConfigurer;
-
-import com.google.common.base.Preconditions;
-
-import org.apache.tez.runtime.library.partitioner.HashPartitioner;
-import org.apache.tez.runtime.library.processor.SimpleProcessor;
-
-/**
- * Simple example to perform WordCount using Tez API's. WordCount is the
- * HelloWorld program of distributed data processing and counts the number
- * of occurrences of a word in a distributed text data set.
- */
-public class WordCount extends Configured implements Tool {
-
- /*
- * Example code to write a processor in Tez.
- * Processors typically apply the main application logic to the data.
- * TokenProcessor tokenizes the input data.
- * It uses an input that provide a Key-Value reader and writes
- * output to a Key-Value writer. The processor inherits from SimpleProcessor
- * since it does not need to handle any advanced constructs for Processors.
- */
- public static class TokenProcessor extends SimpleProcessor {
- IntWritable one = new IntWritable(1);
- Text word = new Text();
-
- public TokenProcessor(ProcessorContext context) {
- super(context);
- }
-
- @Override
- public void run() throws Exception {
- Preconditions.checkArgument(getInputs().size() == 1);
- Preconditions.checkArgument(getOutputs().size() == 1);
- // the recommended approach is to cast the reader/writer to a specific type instead
- // of casting the input/output. This allows the actual input/output type to be replaced
- // without affecting the semantic guarantees of the data type that are represented by
- // the reader and writer.
- KeyValueReader kvReader = (KeyValueReader) getInputs().values().iterator().next().getReader();
- KeyValueWriter kvWriter = (KeyValueWriter) getOutputs().values().iterator().next().getWriter();
- while (kvReader.next()) {
- StringTokenizer itr = new StringTokenizer(kvReader.getCurrentValue().toString());
- while (itr.hasMoreTokens()) {
- word.set(itr.nextToken());
- // Count 1 every time a word is observed. Word is the key a 1 is the value
- kvWriter.write(word, one);
- }
- }
- }
-
- }
-
- /*
- * Example code to write a processor that commits final output to a data sink
- * The SumProcessor aggregates the sum of individual word counts generated by
- * the TokenProcessor.
- * The SumProcessor is connected to a DataSink. In this case, its an Output that
- * writes the data via an OutputFormat to a data sink (typically HDFS). Thats why
- * it derives from SimpleMRProcessor that takes care of handling the necessary
- * output commit operations that makes the final output available for consumers.
- */
- public static class SumProcessor extends SimpleMRProcessor {
- public SumProcessor(ProcessorContext context) {
- super(context);
- }
-
- @Override
- public void run() throws Exception {
- Preconditions.checkArgument(getInputs().size() == 1);
- Preconditions.checkArgument(getOutputs().size() == 1);
- KeyValueWriter kvWriter = (KeyValueWriter) getOutputs().values().iterator().next().getWriter();
- // The KeyValues reader provides all values for a given key. The aggregation of values per key
- // is done by the LogicalInput. Since the key is the word and the values are its counts in
- // the different TokenProcessors, summing all values per key provides the sum for that word.
- KeyValuesReader kvReader = (KeyValuesReader) getInputs().values().iterator().next().getReader();
- while (kvReader.next()) {
- Text word = (Text) kvReader.getCurrentKey();
- int sum = 0;
- for (Object value : kvReader.getCurrentValues()) {
- sum += ((IntWritable) value).get();
- }
- kvWriter.write(word, new IntWritable(sum));
- }
- // deriving from SimpleMRProcessor takes care of committing the output
- // It automatically invokes the commit logic for the OutputFormat if necessary.
- }
- }
-
- private DAG createDAG(TezConfiguration tezConf, String inputPath, String outputPath,
- int numPartitions) throws IOException {
-
- // Create the descriptor that describes the input data to Tez. Using MRInput to read text
- // data from the given input path. The TextInputFormat is used to read the text data.
- DataSourceDescriptor dataSource = MRInput.createConfigurer(new Configuration(tezConf),
- TextInputFormat.class, inputPath).create();
-
- // Create a descriptor that describes the output data to Tez. Using MROoutput to write text
- // data to the given output path. The TextOutputFormat is used to write the text data.
- DataSinkDescriptor dataSink = MROutput.createConfigurer(new Configuration(tezConf),
- TextOutputFormat.class, outputPath).create();
-
- // Create a vertex that reads the data from the data source and tokenizes it using the
- // TokenProcessor. The number of tasks that will do the work for this vertex will be decided
- // using the information provided by the data source descriptor.
- Vertex tokenizerVertex = new Vertex("Tokenizer", new ProcessorDescriptor(
- TokenProcessor.class.getName())).addDataSource("Input", dataSource);
-
- // Create the edge that represents the movement and semantics of data between the producer
- // Tokenizer vertex and the consumer Summation vertex. In order to perform the summation in
- // parallel the tokenized data will be partitioned by word such that a given word goes to the
- // same partition. The counts for the words should be grouped together per word. To achieve this
- // we can use an edge that contains an input/output pair that handles partitioning and grouping
- // of key value data. We use the helper OrderedPartitionedKVEdgeConfigurer to create such an
- // edge. Internally, it sets up matching Tez inputs and outputs that can perform this logic.
- // We specify the key, value and partitioner type. Here the key type is Text (for word), the
- // value type is IntWritable (for count) and we using a hash based partitioner. This is a helper
- // object. The edge can be configured by configuring the input, output etc individually without
- // using this helper.
- OrderedPartitionedKVEdgeConfigurer edgeConf = OrderedPartitionedKVEdgeConfigurer
- .newBuilder(Text.class.getName(), IntWritable.class.getName(),
- HashPartitioner.class.getName()).build();
-
- // Create a vertex that reads the tokenized data and calculates the sum using the SumProcessor.
- // The number of tasks that do the work of this vertex depends on the number of partitions used
- // to distribute the sum processing. In this case, its been made configurable via the
- // numPartitions parameter.
- Vertex summationVertex = new Vertex("Summation",
- new ProcessorDescriptor(SumProcessor.class.getName()), numPartitions)
- .addDataSink("Output", dataSink);
-
- // No need to add jar containing this class as assumed to be part of the Tez jars. Otherwise
- // we would have to add the jars for this code as local files to the vertices.
-
- // Create DAG and add the vertices. Connect the producer and consumer vertices via the edge
- DAG dag = new DAG("WordCount");
- dag.addVertex(tokenizerVertex)
- .addVertex(summationVertex)
- .addEdge(
- new Edge(tokenizerVertex, summationVertex, edgeConf.createDefaultEdgeProperty()));
- return dag;
- }
-
- private static void printUsage() {
- System.err.println("Usage: " + " wordcount in out [numPartitions]");
- ToolRunner.printGenericCommandUsage(System.err);
- }
-
- public boolean run(String inputPath, String outputPath, Configuration conf,
- int numPartitions) throws Exception {
- System.out.println("Running WordCount");
- TezConfiguration tezConf;
- if (conf != null) {
- tezConf = new TezConfiguration(conf);
- } else {
- tezConf = new TezConfiguration();
- }
-
- // Create the TezClient to submit the DAG. Pass the tezConf that has all necessary global and
- // dag specific configurations
- TezClient tezClient = new TezClient("WordCount", tezConf);
- // TezClient must be started before it can be used
- tezClient.start();
-
- try {
- DAG dag = createDAG(tezConf, inputPath, outputPath, numPartitions);
-
- // check that the execution environment is ready
- tezClient.waitTillReady();
- // submit the dag and receive a dag client to monitor the progress
- DAGClient dagClient = tezClient.submitDAG(dag);
-
- // monitor the progress and wait for completion. This method blocks until the dag is done.
- DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(null);
- // check success or failure and print diagnostics
- if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
- System.out.println("WordCount failed with diagnostics: " + dagStatus.getDiagnostics());
- return false;
- }
- return true;
- } finally {
- // stop the client to perform cleanup
- tezClient.stop();
- }
- }
-
- @Override
- public int run(String[] args) throws Exception {
- Configuration conf = getConf();
- String [] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
- if (otherArgs.length < 2 || otherArgs.length > 3) {
- printUsage();
- return 2;
- }
- WordCount job = new WordCount();
- if (job.run(otherArgs[0], otherArgs[1], conf,
- (otherArgs.length == 3 ? Integer.parseInt(otherArgs[2]) : 1))) {
- return 0;
- }
- return 1;
- }
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new WordCount(), args);
- System.exit(res);
- }
-}
[02/10] TEZ-1055. Rename tez-mapreduce-examples to tez-examples
(Hitesh Shah via bikas)
Posted by bi...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
new file mode 100644
index 0000000..72c14fc
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
@@ -0,0 +1,479 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.StringTokenizer;
+import java.util.TreeMap;
+
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileAlreadyExistsException;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.mapreduce.security.TokenCache;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.tez.client.PreWarmVertex;
+import org.apache.tez.client.TezClientUtils;
+import org.apache.tez.client.TezClient;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.DataSourceDescriptor;
+import org.apache.tez.dag.api.Edge;
+import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.api.client.StatusGetOpts;
+import org.apache.tez.mapreduce.examples.helpers.SplitsInClientOptionParser;
+import org.apache.tez.mapreduce.hadoop.MRHelpers;
+import org.apache.tez.mapreduce.hadoop.MRInputHelpers;
+import org.apache.tez.mapreduce.hadoop.MRJobConfig;
+import org.apache.tez.mapreduce.input.MRInputLegacy;
+import org.apache.tez.mapreduce.output.MROutputLegacy;
+import org.apache.tez.mapreduce.processor.map.MapProcessor;
+import org.apache.tez.mapreduce.processor.reduce.ReduceProcessor;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
+import org.apache.tez.runtime.library.conf.OrderedPartitionedKVEdgeConfigurer;
+import org.apache.tez.runtime.library.partitioner.HashPartitioner;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * An MRR job built on top of word count to return words sorted by
+ * their frequency of occurrence.
+ *
+ * Use -DUSE_TEZ_SESSION=true to run jobs in a session mode.
+ * If multiple input/outputs are provided, this job will process each pair
+ * as a separate DAG in a sequential manner.
+ * Use -DINTER_JOB_SLEEP_INTERVAL=<N> where N is the sleep interval in seconds
+ * between the sequential DAGs.
+ */
+public class TestOrderedWordCount extends Configured implements Tool {
+
+ private static Log LOG = LogFactory.getLog(TestOrderedWordCount.class);
+
+ public static class TokenizerMapper
+ extends Mapper<Object, Text, Text, IntWritable>{
+
+ private final static IntWritable one = new IntWritable(1);
+ private Text word = new Text();
+
+ public void map(Object key, Text value, Context context
+ ) throws IOException, InterruptedException {
+ StringTokenizer itr = new StringTokenizer(value.toString());
+ while (itr.hasMoreTokens()) {
+ word.set(itr.nextToken());
+ context.write(word, one);
+ }
+ }
+ }
+
+ public static class IntSumReducer
+ extends Reducer<Text,IntWritable,IntWritable, Text> {
+ private IntWritable result = new IntWritable();
+
+ public void reduce(Text key, Iterable<IntWritable> values,
+ Context context
+ ) throws IOException, InterruptedException {
+ int sum = 0;
+ for (IntWritable val : values) {
+ sum += val.get();
+ }
+ result.set(sum);
+ context.write(result, key);
+ }
+ }
+
+ /**
+ * Shuffle ensures ordering based on count of employees per department
+ * hence the final reducer is a no-op and just emits the department name
+ * with the employee count per department.
+ */
+ public static class MyOrderByNoOpReducer
+ extends Reducer<IntWritable, Text, Text, IntWritable> {
+
+ public void reduce(IntWritable key, Iterable<Text> values,
+ Context context
+ ) throws IOException, InterruptedException {
+ for (Text word : values) {
+ context.write(word, key);
+ }
+ }
+ }
+
+ private Credentials credentials = new Credentials();
+
+ @VisibleForTesting
+ public DAG createDAG(FileSystem fs, Configuration conf,
+ Map<String, LocalResource> commonLocalResources, Path stagingDir,
+ int dagIndex, String inputPath, String outputPath,
+ boolean generateSplitsInClient) throws Exception {
+
+ Configuration mapStageConf = new JobConf(conf);
+ mapStageConf.set(MRJobConfig.MAP_CLASS_ATTR,
+ TokenizerMapper.class.getName());
+
+ MRHelpers.translateMRConfToTez(mapStageConf);
+
+ Configuration iReduceStageConf = new JobConf(conf);
+ // TODO replace with auto-reduce parallelism
+ iReduceStageConf.setInt(MRJobConfig.NUM_REDUCES, 2);
+ iReduceStageConf.set(MRJobConfig.REDUCE_CLASS_ATTR,
+ IntSumReducer.class.getName());
+ iReduceStageConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, Text.class.getName());
+ iReduceStageConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS,
+ IntWritable.class.getName());
+ iReduceStageConf.setBoolean("mapred.mapper.new-api", true);
+ MRHelpers.translateMRConfToTez(iReduceStageConf);
+
+ Configuration finalReduceConf = new JobConf(conf);
+ finalReduceConf.setInt(MRJobConfig.NUM_REDUCES, 1);
+ finalReduceConf.set(MRJobConfig.REDUCE_CLASS_ATTR,
+ MyOrderByNoOpReducer.class.getName());
+ finalReduceConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, IntWritable.class.getName());
+ finalReduceConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, Text.class.getName());
+ MRHelpers.translateMRConfToTez(finalReduceConf);
+
+ MRHelpers.configureMRApiUsage(mapStageConf);
+ MRHelpers.configureMRApiUsage(iReduceStageConf);
+ MRHelpers.configureMRApiUsage(finalReduceConf);
+
+ List<Vertex> vertices = new ArrayList<Vertex>();
+
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream(4096);
+ mapStageConf.writeXml(outputStream);
+ String mapStageHistoryText = new String(outputStream.toByteArray(), "UTF-8");
+ DataSourceDescriptor dsd;
+ if (generateSplitsInClient) {
+ mapStageConf.set(MRJobConfig.INPUT_FORMAT_CLASS_ATTR,
+ TextInputFormat.class.getName());
+ mapStageConf.set(FileInputFormat.INPUT_DIR, inputPath);
+ mapStageConf.setBoolean("mapred.mapper.new-api", true);
+ dsd = MRInputHelpers.configureMRInputWithLegacySplitGeneration(mapStageConf, stagingDir, true);
+ } else {
+ dsd = MRInputLegacy.createConfigurer(mapStageConf, TextInputFormat.class, inputPath).create();
+ }
+
+ Vertex mapVertex = new Vertex("initialmap", new ProcessorDescriptor(
+ MapProcessor.class.getName()).setUserPayload(
+ TezUtils.createUserPayloadFromConf(mapStageConf))
+ .setHistoryText(mapStageHistoryText)).setTaskLocalFiles(commonLocalResources);
+ mapVertex.addDataSource("MRInput", dsd);
+ vertices.add(mapVertex);
+
+ ByteArrayOutputStream iROutputStream = new ByteArrayOutputStream(4096);
+ iReduceStageConf.writeXml(iROutputStream);
+ String iReduceStageHistoryText = new String(iROutputStream.toByteArray(), "UTF-8");
+ Vertex ivertex = new Vertex("intermediate_reducer", new ProcessorDescriptor(
+ ReduceProcessor.class.getName())
+ .setUserPayload(TezUtils.createUserPayloadFromConf(iReduceStageConf))
+ .setHistoryText(iReduceStageHistoryText), 2);
+ ivertex.setTaskLocalFiles(commonLocalResources);
+ vertices.add(ivertex);
+
+ ByteArrayOutputStream finalReduceOutputStream = new ByteArrayOutputStream(4096);
+ finalReduceConf.writeXml(finalReduceOutputStream);
+ String finalReduceStageHistoryText = new String(finalReduceOutputStream.toByteArray(), "UTF-8");
+ UserPayload finalReducePayload = TezUtils.createUserPayloadFromConf(finalReduceConf);
+ Vertex finalReduceVertex = new Vertex("finalreduce",
+ new ProcessorDescriptor(
+ ReduceProcessor.class.getName())
+ .setUserPayload(finalReducePayload)
+ .setHistoryText(finalReduceStageHistoryText), 1);
+ finalReduceVertex.setTaskLocalFiles(commonLocalResources);
+ finalReduceVertex.addDataSink("MROutput",
+ MROutputLegacy.createConfigurer(finalReduceConf, TextOutputFormat.class, outputPath)
+ .create());
+ vertices.add(finalReduceVertex);
+
+ DAG dag = new DAG("OrderedWordCount" + dagIndex);
+ for (int i = 0; i < vertices.size(); ++i) {
+ dag.addVertex(vertices.get(i));
+ }
+
+ OrderedPartitionedKVEdgeConfigurer edgeConf1 = OrderedPartitionedKVEdgeConfigurer
+ .newBuilder(Text.class.getName(), IntWritable.class.getName(),
+ HashPartitioner.class.getName()).setFromConfiguration(conf)
+ .configureInput().useLegacyInput().done().build();
+ dag.addEdge(
+ new Edge(dag.getVertex("initialmap"), dag.getVertex("intermediate_reducer"),
+ edgeConf1.createDefaultEdgeProperty()));
+
+ OrderedPartitionedKVEdgeConfigurer edgeConf2 = OrderedPartitionedKVEdgeConfigurer
+ .newBuilder(IntWritable.class.getName(), Text.class.getName(),
+ HashPartitioner.class.getName()).setFromConfiguration(conf)
+ .configureInput().useLegacyInput().done().build();
+ dag.addEdge(
+ new Edge(dag.getVertex("intermediate_reducer"), dag.getVertex("finalreduce"),
+ edgeConf2.createDefaultEdgeProperty()));
+
+ return dag;
+ }
+
+ private static void printUsage() {
+ String options = " [-generateSplitsInClient true/<false>]";
+ System.err.println("Usage: testorderedwordcount <in> <out>" + options);
+ System.err.println("Usage (In Session Mode):"
+ + " testorderedwordcount <in1> <out1> ... <inN> <outN>" + options);
+ ToolRunner.printGenericCommandUsage(System.err);
+ }
+
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Configuration conf = getConf();
+ String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+
+ boolean generateSplitsInClient;
+
+ SplitsInClientOptionParser splitCmdLineParser = new SplitsInClientOptionParser();
+ try {
+ generateSplitsInClient = splitCmdLineParser.parse(otherArgs, false);
+ otherArgs = splitCmdLineParser.getRemainingArgs();
+ } catch (ParseException e1) {
+ System.err.println("Invalid options");
+ printUsage();
+ return 2;
+ }
+
+ boolean useTezSession = conf.getBoolean("USE_TEZ_SESSION", true);
+ long interJobSleepTimeout = conf.getInt("INTER_JOB_SLEEP_INTERVAL", 0)
+ * 1000;
+
+ boolean retainStagingDir = conf.getBoolean("RETAIN_STAGING_DIR", false);
+
+ if (((otherArgs.length%2) != 0)
+ || (!useTezSession && otherArgs.length != 2)) {
+ printUsage();
+ return 2;
+ }
+
+ List<String> inputPaths = new ArrayList<String>();
+ List<String> outputPaths = new ArrayList<String>();
+
+ for (int i = 0; i < otherArgs.length; i+=2) {
+ inputPaths.add(otherArgs[i]);
+ outputPaths.add(otherArgs[i+1]);
+ }
+
+ UserGroupInformation.setConfiguration(conf);
+
+ TezConfiguration tezConf = new TezConfiguration(conf);
+ TestOrderedWordCount instance = new TestOrderedWordCount();
+
+ FileSystem fs = FileSystem.get(conf);
+
+ String stagingDirStr = conf.get(TezConfiguration.TEZ_AM_STAGING_DIR,
+ TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT) + Path.SEPARATOR +
+ Long.toString(System.currentTimeMillis());
+ Path stagingDir = new Path(stagingDirStr);
+ FileSystem pathFs = stagingDir.getFileSystem(tezConf);
+ pathFs.mkdirs(new Path(stagingDirStr));
+
+ tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr);
+ stagingDir = pathFs.makeQualified(new Path(stagingDirStr));
+
+ TokenCache.obtainTokensForNamenodes(instance.credentials, new Path[] {stagingDir}, conf);
+ TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
+
+ // No need to add jar containing this class as assumed to be part of
+ // the tez jars.
+
+ // TEZ-674 Obtain tokens based on the Input / Output paths. For now assuming staging dir
+ // is the same filesystem as the one used for Input/Output.
+
+ if (useTezSession) {
+ LOG.info("Creating Tez Session");
+ tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
+ } else {
+ tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
+ }
+ TezClient tezSession = new TezClient("OrderedWordCountSession", tezConf,
+ null, instance.credentials);
+ tezSession.start();
+
+ DAGStatus dagStatus = null;
+ DAGClient dagClient = null;
+ String[] vNames = { "initialmap", "intermediate_reducer",
+ "finalreduce" };
+
+ Set<StatusGetOpts> statusGetOpts = EnumSet.of(StatusGetOpts.GET_COUNTERS);
+ try {
+ for (int dagIndex = 1; dagIndex <= inputPaths.size(); ++dagIndex) {
+ if (dagIndex != 1
+ && interJobSleepTimeout > 0) {
+ try {
+ LOG.info("Sleeping between jobs, sleepInterval="
+ + (interJobSleepTimeout/1000));
+ Thread.sleep(interJobSleepTimeout);
+ } catch (InterruptedException e) {
+ LOG.info("Main thread interrupted. Breaking out of job loop");
+ break;
+ }
+ }
+
+ String inputPath = inputPaths.get(dagIndex-1);
+ String outputPath = outputPaths.get(dagIndex-1);
+
+ if (fs.exists(new Path(outputPath))) {
+ throw new FileAlreadyExistsException("Output directory "
+ + outputPath + " already exists");
+ }
+ LOG.info("Running OrderedWordCount DAG"
+ + ", dagIndex=" + dagIndex
+ + ", inputPath=" + inputPath
+ + ", outputPath=" + outputPath);
+
+ Map<String, LocalResource> localResources =
+ new TreeMap<String, LocalResource>();
+
+ DAG dag = instance.createDAG(fs, conf, localResources,
+ stagingDir, dagIndex, inputPath, outputPath,
+ generateSplitsInClient);
+
+ boolean doPreWarm = dagIndex == 1 && useTezSession
+ && conf.getBoolean("PRE_WARM_SESSION", true);
+ int preWarmNumContainers = 0;
+ if (doPreWarm) {
+ preWarmNumContainers = conf.getInt("PRE_WARM_NUM_CONTAINERS", 0);
+ if (preWarmNumContainers <= 0) {
+ doPreWarm = false;
+ }
+ }
+ if (doPreWarm) {
+ LOG.info("Pre-warming Session");
+ PreWarmVertex preWarmVertex = new PreWarmVertex("PreWarm", preWarmNumContainers, dag
+ .getVertex("initialmap").getTaskResource());
+ preWarmVertex.setTaskLocalFiles(dag.getVertex("initialmap").getTaskLocalFiles());
+ preWarmVertex.setTaskEnvironment(dag.getVertex("initialmap").getTaskEnvironment());
+ preWarmVertex.setTaskLaunchCmdOpts(dag.getVertex("initialmap").getTaskLaunchCmdOpts());
+
+ tezSession.preWarm(preWarmVertex);
+ }
+
+ if (useTezSession) {
+ LOG.info("Waiting for TezSession to get into ready state");
+ waitForTezSessionReady(tezSession);
+ LOG.info("Submitting DAG to Tez Session, dagIndex=" + dagIndex);
+ dagClient = tezSession.submitDAG(dag);
+ LOG.info("Submitted DAG to Tez Session, dagIndex=" + dagIndex);
+ } else {
+ LOG.info("Submitting DAG as a new Tez Application");
+ dagClient = tezSession.submitDAG(dag);
+ }
+
+ while (true) {
+ dagStatus = dagClient.getDAGStatus(statusGetOpts);
+ if (dagStatus.getState() == DAGStatus.State.RUNNING ||
+ dagStatus.getState() == DAGStatus.State.SUCCEEDED ||
+ dagStatus.getState() == DAGStatus.State.FAILED ||
+ dagStatus.getState() == DAGStatus.State.KILLED ||
+ dagStatus.getState() == DAGStatus.State.ERROR) {
+ break;
+ }
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ // continue;
+ }
+ }
+
+
+ while (dagStatus.getState() != DAGStatus.State.SUCCEEDED &&
+ dagStatus.getState() != DAGStatus.State.FAILED &&
+ dagStatus.getState() != DAGStatus.State.KILLED &&
+ dagStatus.getState() != DAGStatus.State.ERROR) {
+ if (dagStatus.getState() == DAGStatus.State.RUNNING) {
+ ExampleDriver.printDAGStatus(dagClient, vNames);
+ }
+ try {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ // continue;
+ }
+ dagStatus = dagClient.getDAGStatus(statusGetOpts);
+ } catch (TezException e) {
+ LOG.fatal("Failed to get application progress. Exiting");
+ return -1;
+ }
+ }
+ ExampleDriver.printDAGStatus(dagClient, vNames,
+ true, true);
+ LOG.info("DAG " + dagIndex + " completed. "
+ + "FinalState=" + dagStatus.getState());
+ if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
+ LOG.info("DAG " + dagIndex + " diagnostics: "
+ + dagStatus.getDiagnostics());
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("Error occurred when submitting/running DAGs", e);
+ throw e;
+ } finally {
+ if (!retainStagingDir) {
+ pathFs.delete(stagingDir, true);
+ }
+ LOG.info("Shutting down session");
+ tezSession.stop();
+ }
+
+ if (!useTezSession) {
+ ExampleDriver.printDAGStatus(dagClient, vNames);
+ LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
+ }
+ return dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1;
+ }
+
+ private static void waitForTezSessionReady(TezClient tezSession)
+ throws IOException, TezException, InterruptedException {
+ tezSession.waitTillReady();
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(), new TestOrderedWordCount(), args);
+ System.exit(res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java
new file mode 100644
index 0000000..a9b23e0
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java
@@ -0,0 +1,295 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.mapreduce.examples;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.StringTokenizer;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileAlreadyExistsException;
+import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.mapred.TextOutputFormat;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.tez.client.TezClient;
+import org.apache.tez.dag.api.DataSinkDescriptor;
+import org.apache.tez.dag.api.DataSourceDescriptor;
+import org.apache.tez.dag.api.GroupInputEdge;
+import org.apache.tez.dag.api.VertexGroup;
+import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.Edge;
+import org.apache.tez.dag.api.InputDescriptor;
+import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.client.DAGClient;
+import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.api.client.StatusGetOpts;
+import org.apache.tez.mapreduce.input.MRInput;
+import org.apache.tez.mapreduce.input.MRInput.MRInputConfigurer;
+import org.apache.tez.mapreduce.output.MROutput;
+import org.apache.tez.mapreduce.processor.SimpleMRProcessor;
+import org.apache.tez.runtime.api.LogicalInput;
+import org.apache.tez.runtime.api.Output;
+import org.apache.tez.runtime.api.ProcessorContext;
+import org.apache.tez.runtime.library.api.KeyValueReader;
+import org.apache.tez.runtime.library.api.KeyValueWriter;
+import org.apache.tez.runtime.library.api.KeyValuesReader;
+import org.apache.tez.runtime.library.conf.OrderedPartitionedKVEdgeConfigurer;
+import org.apache.tez.runtime.library.input.ConcatenatedMergedKeyValuesInput;
+import org.apache.tez.runtime.library.partitioner.HashPartitioner;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+
+public class UnionExample {
+
+ public static class TokenProcessor extends SimpleMRProcessor {
+ IntWritable one = new IntWritable(1);
+ Text word = new Text();
+
+ public TokenProcessor(ProcessorContext context) {
+ super(context);
+ }
+
+ @Override
+ public void run() throws Exception {
+ Preconditions.checkArgument(getInputs().size() == 1);
+ boolean inUnion = true;
+ if (getContext().getTaskVertexName().equals("map3")) {
+ inUnion = false;
+ }
+ Preconditions.checkArgument(getOutputs().size() == (inUnion ? 2 : 1));
+ Preconditions.checkArgument(getOutputs().containsKey("checker"));
+ MRInput input = (MRInput) getInputs().values().iterator().next();
+ KeyValueReader kvReader = input.getReader();
+ Output output = getOutputs().get("checker");
+ KeyValueWriter kvWriter = (KeyValueWriter) output.getWriter();
+ MROutput parts = null;
+ KeyValueWriter partsWriter = null;
+ if (inUnion) {
+ parts = (MROutput) getOutputs().get("parts");
+ partsWriter = parts.getWriter();
+ }
+ while (kvReader.next()) {
+ StringTokenizer itr = new StringTokenizer(kvReader.getCurrentValue().toString());
+ while (itr.hasMoreTokens()) {
+ word.set(itr.nextToken());
+ kvWriter.write(word, one);
+ if (inUnion) {
+ partsWriter.write(word, one);
+ }
+ }
+ }
+ }
+
+ }
+
+ public static class UnionProcessor extends SimpleMRProcessor {
+ IntWritable one = new IntWritable(1);
+
+ public UnionProcessor(ProcessorContext context) {
+ super(context);
+ }
+
+ @Override
+ public void run() throws Exception {
+ Preconditions.checkArgument(getInputs().size() == 2);
+ Preconditions.checkArgument(getOutputs().size() == 2);
+ MROutput out = (MROutput) getOutputs().get("union");
+ MROutput allParts = (MROutput) getOutputs().get("all-parts");
+ KeyValueWriter kvWriter = out.getWriter();
+ KeyValueWriter partsWriter = allParts.getWriter();
+ Map<String, AtomicInteger> unionKv = Maps.newHashMap();
+ LogicalInput union = getInputs().get("union");
+ KeyValuesReader kvReader = (KeyValuesReader) union.getReader();
+ while (kvReader.next()) {
+ String word = ((Text) kvReader.getCurrentKey()).toString();
+ IntWritable intVal = (IntWritable) kvReader.getCurrentValues().iterator().next();
+ for (int i = 0; i < intVal.get(); ++i) {
+ partsWriter.write(word, one);
+ }
+ AtomicInteger value = unionKv.get(word);
+ if (value == null) {
+ unionKv.put(word, new AtomicInteger(intVal.get()));
+ } else {
+ value.addAndGet(intVal.get());
+ }
+ }
+ LogicalInput map3 = getInputs().get("map3");
+ kvReader = (KeyValuesReader) map3.getReader();
+ while (kvReader.next()) {
+ String word = ((Text) kvReader.getCurrentKey()).toString();
+ IntWritable intVal = (IntWritable) kvReader.getCurrentValues().iterator().next();
+ AtomicInteger value = unionKv.get(word);
+ if (value == null) {
+ throw new TezUncheckedException("Expected to exist: " + word);
+ } else {
+ value.getAndAdd(intVal.get() * -2);
+ }
+ }
+ for (AtomicInteger value : unionKv.values()) {
+ if (value.get() != 0) {
+ throw new TezUncheckedException("Unexpected non-zero value");
+ }
+ }
+ kvWriter.write("Union", new IntWritable(unionKv.size()));
+ }
+
+ }
+
+ private DAG createDAG(FileSystem fs, TezConfiguration tezConf,
+ Map<String, LocalResource> localResources, Path stagingDir,
+ String inputPath, String outputPath) throws IOException {
+ DAG dag = new DAG("UnionExample");
+
+ int numMaps = -1;
+ Configuration inputConf = new Configuration(tezConf);
+ MRInputConfigurer configurer = MRInput.createConfigurer(inputConf, TextInputFormat.class,
+ inputPath);
+ DataSourceDescriptor dataSource = configurer.generateSplitsInAM(false).create();
+
+ Vertex mapVertex1 = new Vertex("map1", new ProcessorDescriptor(
+ TokenProcessor.class.getName()), numMaps).addDataSource("MRInput", dataSource);
+
+ Vertex mapVertex2 = new Vertex("map2", new ProcessorDescriptor(
+ TokenProcessor.class.getName()), numMaps).addDataSource("MRInput", dataSource);
+
+ Vertex mapVertex3 = new Vertex("map3", new ProcessorDescriptor(
+ TokenProcessor.class.getName()), numMaps).addDataSource("MRInput", dataSource);
+
+ Vertex checkerVertex = new Vertex("checker", new ProcessorDescriptor(
+ UnionProcessor.class.getName()), 1);
+
+ Configuration outputConf = new Configuration(tezConf);
+ DataSinkDescriptor od = MROutput.createConfigurer(outputConf,
+ TextOutputFormat.class, outputPath).create();
+ checkerVertex.addDataSink("union", od);
+
+
+ Configuration allPartsConf = new Configuration(tezConf);
+ DataSinkDescriptor od2 = MROutput.createConfigurer(allPartsConf,
+ TextOutputFormat.class, outputPath + "-all-parts").create();
+ checkerVertex.addDataSink("all-parts", od2);
+
+ Configuration partsConf = new Configuration(tezConf);
+ DataSinkDescriptor od1 = MROutput.createConfigurer(partsConf,
+ TextOutputFormat.class, outputPath + "-parts").create();
+ VertexGroup unionVertex = dag.createVertexGroup("union", mapVertex1, mapVertex2);
+ unionVertex.addDataSink("parts", od1);
+
+ OrderedPartitionedKVEdgeConfigurer edgeConf = OrderedPartitionedKVEdgeConfigurer
+ .newBuilder(Text.class.getName(), IntWritable.class.getName(),
+ HashPartitioner.class.getName()).build();
+
+ dag.addVertex(mapVertex1)
+ .addVertex(mapVertex2)
+ .addVertex(mapVertex3)
+ .addVertex(checkerVertex)
+ .addEdge(
+ new Edge(mapVertex3, checkerVertex, edgeConf.createDefaultEdgeProperty()))
+ .addEdge(
+ new GroupInputEdge(unionVertex, checkerVertex, edgeConf.createDefaultEdgeProperty(),
+ new InputDescriptor(
+ ConcatenatedMergedKeyValuesInput.class.getName())));
+ return dag;
+ }
+
+ private static void printUsage() {
+ System.err.println("Usage: " + " unionexample <in1> <out1>");
+ }
+
+ public boolean run(String inputPath, String outputPath, Configuration conf) throws Exception {
+ System.out.println("Running UnionExample");
+ // conf and UGI
+ TezConfiguration tezConf;
+ if (conf != null) {
+ tezConf = new TezConfiguration(conf);
+ } else {
+ tezConf = new TezConfiguration();
+ }
+ UserGroupInformation.setConfiguration(tezConf);
+ String user = UserGroupInformation.getCurrentUser().getShortUserName();
+
+ // staging dir
+ FileSystem fs = FileSystem.get(tezConf);
+ String stagingDirStr = Path.SEPARATOR + "user" + Path.SEPARATOR
+ + user + Path.SEPARATOR+ ".staging" + Path.SEPARATOR
+ + Path.SEPARATOR + Long.toString(System.currentTimeMillis());
+ Path stagingDir = new Path(stagingDirStr);
+ tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr);
+ stagingDir = fs.makeQualified(stagingDir);
+
+
+ // No need to add jar containing this class as assumed to be part of
+ // the tez jars.
+
+ // TEZ-674 Obtain tokens based on the Input / Output paths. For now assuming staging dir
+ // is the same filesystem as the one used for Input/Output.
+
+ TezClient tezSession = new TezClient("UnionExampleSession", tezConf);
+ tezSession.start();
+
+ DAGClient dagClient = null;
+
+ try {
+ if (fs.exists(new Path(outputPath))) {
+ throw new FileAlreadyExistsException("Output directory "
+ + outputPath + " already exists");
+ }
+
+ Map<String, LocalResource> localResources =
+ new TreeMap<String, LocalResource>();
+
+ DAG dag = createDAG(fs, tezConf, localResources,
+ stagingDir, inputPath, outputPath);
+
+ tezSession.waitTillReady();
+ dagClient = tezSession.submitDAG(dag);
+
+ // monitoring
+ DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(EnumSet.of(StatusGetOpts.GET_COUNTERS));
+ if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
+ System.out.println("DAG diagnostics: " + dagStatus.getDiagnostics());
+ return false;
+ }
+ return true;
+ } finally {
+ fs.delete(stagingDir, true);
+ tezSession.stop();
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ if (args.length != 2) {
+ printUsage();
+ System.exit(2);
+ }
+ UnionExample job = new UnionExample();
+ job.run(args[0], args[1], null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/helpers/SplitsInClientOptionParser.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/helpers/SplitsInClientOptionParser.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/helpers/SplitsInClientOptionParser.java
new file mode 100644
index 0000000..93ec860
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/helpers/SplitsInClientOptionParser.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.examples.helpers;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+import com.google.common.base.Preconditions;
+
+public class SplitsInClientOptionParser {
+
+ private CommandLine cmdLine;
+ private String[] otherArgs;
+
+ private boolean parsed = false;
+
+ public SplitsInClientOptionParser() {
+
+ }
+
+ public String[] getRemainingArgs() {
+ Preconditions.checkState(parsed,
+ "Cannot get remaining args without parsing");
+ return otherArgs;
+ }
+
+ @SuppressWarnings("static-access")
+ public boolean parse(String[] args, boolean defaultVal) throws ParseException {
+ Preconditions.checkState(parsed == false,
+ "Craete a new instance for different option sets");
+ parsed = true;
+ Options opts = new Options();
+ Option opt = OptionBuilder
+ .withArgName("splits_in_client")
+ .hasArg()
+ .withDescription(
+ "specify whether splits should be generated in the client")
+ .create("generateSplitsInClient");
+ opts.addOption(opt);
+ CommandLineParser parser = new GnuParser();
+
+ cmdLine = parser.parse(opts, args, false);
+ if (cmdLine.hasOption("generateSplitsInClient")) {
+ defaultVal = Boolean.parseBoolean(cmdLine
+ .getOptionValue("generateSplitsInClient"));
+ }
+ otherArgs = cmdLine.getArgs();
+ return defaultVal;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordInputProcessor.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordInputProcessor.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordInputProcessor.java
new file mode 100644
index 0000000..aa7b836
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordInputProcessor.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.processor;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.mapreduce.examples.FilterLinesByWord;
+import org.apache.tez.mapreduce.examples.FilterLinesByWord.TextLongPair;
+import org.apache.tez.mapreduce.hadoop.MRJobConfig;
+import org.apache.tez.mapreduce.input.MRInput;
+import org.apache.tez.mapreduce.input.MRInputLegacy;
+import org.apache.tez.runtime.api.AbstractLogicalIOProcessor;
+import org.apache.tez.runtime.api.Event;
+import org.apache.tez.runtime.api.LogicalInput;
+import org.apache.tez.runtime.api.LogicalOutput;
+import org.apache.tez.runtime.api.ProcessorContext;
+import org.apache.tez.runtime.library.api.KeyValueReader;
+import org.apache.tez.runtime.library.api.KeyValueWriter;
+import org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput;
+
+public class FilterByWordInputProcessor extends AbstractLogicalIOProcessor {
+
+ private static final Log LOG = LogFactory.getLog(FilterByWordInputProcessor.class);
+
+ private String filterWord;
+
+ public FilterByWordInputProcessor(ProcessorContext context) {
+ super(context);
+ }
+
+
+ @Override
+ public void initialize() throws Exception {
+ Configuration conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());
+ filterWord = conf.get(FilterLinesByWord.FILTER_PARAM_NAME);
+ if (filterWord == null) {
+ getContext().fatalError(null, "No filter word specified");
+ }
+ }
+
+ @Override
+ public void handleEvents(List<Event> processorEvents) {
+ throw new UnsupportedOperationException("Not expecting any events to the broadcast processor");
+
+ }
+
+ @Override
+ public void close() throws Exception {
+ LOG.info("Broadcast Processor closing. Nothing to do");
+ }
+
+ @Override
+ public void run(Map<String, LogicalInput> inputs,
+ Map<String, LogicalOutput> outputs) throws Exception {
+
+ if (inputs.size() != 1) {
+ throw new IllegalStateException("FilterByWordInputProcessor processor can only work with a single input");
+ }
+
+ if (outputs.size() != 1) {
+ throw new IllegalStateException("FilterByWordInputProcessor processor can only work with a single output");
+ }
+
+ for (LogicalInput input : inputs.values()) {
+ input.start();
+ }
+ for (LogicalOutput output : outputs.values()) {
+ output.start();
+ }
+
+ LogicalInput li = inputs.values().iterator().next();
+ if (! (li instanceof MRInput)) {
+ throw new IllegalStateException("FilterByWordInputProcessor processor can only work with MRInput");
+ }
+
+ LogicalOutput lo = outputs.values().iterator().next();
+ if (! (lo instanceof OnFileUnorderedKVOutput)) {
+ throw new IllegalStateException("FilterByWordInputProcessor processor can only work with OnFileUnorderedKVOutput");
+ }
+
+ MRInputLegacy mrInput = (MRInputLegacy) li;
+ mrInput.init();
+ OnFileUnorderedKVOutput kvOutput = (OnFileUnorderedKVOutput) lo;
+
+ Configuration updatedConf = mrInput.getConfigUpdates();
+ Text srcFile = new Text();
+ srcFile.set("UNKNOWN_FILENAME_IN_PROCESSOR");
+ if (updatedConf != null) {
+ String fileName = updatedConf.get(MRJobConfig.MAP_INPUT_FILE);
+ if (fileName != null) {
+ LOG.info("Processing file: " + fileName);
+ srcFile.set(fileName);
+ }
+ }
+
+ KeyValueReader kvReader = mrInput.getReader();
+ KeyValueWriter kvWriter = kvOutput.getWriter();
+
+ while (kvReader.next()) {
+ Object key = kvReader.getCurrentKey();
+ Object val = kvReader.getCurrentValue();
+
+ Text valText = (Text) val;
+ String readVal = valText.toString();
+ if (readVal.contains(filterWord)) {
+ LongWritable lineNum = (LongWritable) key;
+ TextLongPair outVal = new TextLongPair(srcFile, lineNum);
+ kvWriter.write(valText, outVal);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordOutputProcessor.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordOutputProcessor.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordOutputProcessor.java
new file mode 100644
index 0000000..9eca13f
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordOutputProcessor.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.processor;
+
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.tez.mapreduce.output.MROutput;
+import org.apache.tez.mapreduce.processor.SimpleMRProcessor;
+import org.apache.tez.mapreduce.processor.map.MapProcessor;
+import org.apache.tez.runtime.api.Event;
+import org.apache.tez.runtime.api.LogicalInput;
+import org.apache.tez.runtime.api.LogicalOutput;
+import org.apache.tez.runtime.api.ProcessorContext;
+import org.apache.tez.runtime.library.api.KeyValueReader;
+import org.apache.tez.runtime.library.api.KeyValueWriter;
+import org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput;
+
+
+public class FilterByWordOutputProcessor extends SimpleMRProcessor {
+
+ private static final Log LOG = LogFactory.getLog(MapProcessor.class);
+
+ public FilterByWordOutputProcessor(ProcessorContext context) {
+ super(context);
+ }
+
+
+ @Override
+ public void handleEvents(List<Event> processorEvents) {
+ throw new UnsupportedOperationException("Not expecting any events to the broadcast output processor");
+ }
+
+ @Override
+ public void close() throws Exception {
+ LOG.info("Broadcast Output Processor closing. Nothing to do");
+ }
+
+ @Override
+ public void run() throws Exception {
+
+ if (inputs.size() != 1) {
+ throw new IllegalStateException("FilterByWordOutputProcessor processor can only work with a single input");
+ }
+
+ if (outputs.size() != 1) {
+ throw new IllegalStateException("FilterByWordOutputProcessor processor can only work with a single output");
+ }
+
+ for (LogicalInput input : inputs.values()) {
+ input.start();
+ }
+ for (LogicalOutput output : outputs.values()) {
+ output.start();
+ }
+
+ LogicalInput li = inputs.values().iterator().next();
+ if (! (li instanceof ShuffledUnorderedKVInput)) {
+ throw new IllegalStateException("FilterByWordOutputProcessor processor can only work with ShuffledUnorderedKVInput");
+ }
+
+ LogicalOutput lo = outputs.values().iterator().next();
+ if (! (lo instanceof MROutput)) {
+ throw new IllegalStateException("FilterByWordOutputProcessor processor can only work with MROutput");
+ }
+
+ ShuffledUnorderedKVInput kvInput = (ShuffledUnorderedKVInput) li;
+ MROutput mrOutput = (MROutput) lo;
+
+ KeyValueReader kvReader = kvInput.getReader();
+ KeyValueWriter kvWriter = mrOutput.getWriter();
+ while (kvReader.next()) {
+ Object key = kvReader.getCurrentKey();
+ Object value = kvReader.getCurrentValue();
+
+ kvWriter.write(key, value);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/GenSort.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/GenSort.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/GenSort.java
new file mode 100644
index 0000000..c8517e5
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/GenSort.java
@@ -0,0 +1,251 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.mapreduce.examples.terasort;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.math.BigInteger;
+import java.util.zip.Checksum;
+
+import org.apache.hadoop.util.PureJavaCrc32;
+
+/**
+ * A single process data generator for the terasort data. Based on gensort.c
+ * version 1.1 (3 Mar 2009) from Chris Nyberg <ch...@ordinal.com>.
+ */
+public class GenSort {
+
+ /**
+ * Generate a "binary" record suitable for all sort benchmarks *except*
+ * PennySort.
+ */
+ static void generateRecord(byte[] recBuf, Unsigned16 rand,
+ Unsigned16 recordNumber) {
+ /* generate the 10-byte key using the high 10 bytes of the 128-bit
+ * random number
+ */
+ for(int i=0; i < 10; ++i) {
+ recBuf[i] = rand.getByte(i);
+ }
+
+ /* add 2 bytes of "break" */
+ recBuf[10] = 0x00;
+ recBuf[11] = 0x11;
+
+ /* convert the 128-bit record number to 32 bits of ascii hexadecimal
+ * as the next 32 bytes of the record.
+ */
+ for (int i = 0; i < 32; i++) {
+ recBuf[12 + i] = (byte) recordNumber.getHexDigit(i);
+ }
+
+ /* add 4 bytes of "break" data */
+ recBuf[44] = (byte) 0x88;
+ recBuf[45] = (byte) 0x99;
+ recBuf[46] = (byte) 0xAA;
+ recBuf[47] = (byte) 0xBB;
+
+ /* add 48 bytes of filler based on low 48 bits of random number */
+ for(int i=0; i < 12; ++i) {
+ recBuf[48+i*4] = recBuf[49+i*4] = recBuf[50+i*4] = recBuf[51+i*4] =
+ (byte) rand.getHexDigit(20 + i);
+ }
+
+ /* add 4 bytes of "break" data */
+ recBuf[96] = (byte) 0xCC;
+ recBuf[97] = (byte) 0xDD;
+ recBuf[98] = (byte) 0xEE;
+ recBuf[99] = (byte) 0xFF;
+ }
+
+
+ private static BigInteger makeBigInteger(long x) {
+ byte[] data = new byte[8];
+ for(int i=0; i < 8; ++i) {
+ data[i] = (byte) (x >>> (56 - 8*i));
+ }
+ return new BigInteger(1, data);
+ }
+
+ private static final BigInteger NINETY_FIVE = new BigInteger("95");
+
+ /**
+ * Generate an ascii record suitable for all sort benchmarks including
+ * PennySort.
+ */
+ static void generateAsciiRecord(byte[] recBuf, Unsigned16 rand,
+ Unsigned16 recordNumber) {
+
+ /* generate the 10-byte ascii key using mostly the high 64 bits.
+ */
+ long temp = rand.getHigh8();
+ if (temp < 0) {
+ // use biginteger to avoid the negative sign problem
+ BigInteger bigTemp = makeBigInteger(temp);
+ recBuf[0] = (byte) (' ' + (bigTemp.mod(NINETY_FIVE).longValue()));
+ temp = bigTemp.divide(NINETY_FIVE).longValue();
+ } else {
+ recBuf[0] = (byte) (' ' + (temp % 95));
+ temp /= 95;
+ }
+ for(int i=1; i < 8; ++i) {
+ recBuf[i] = (byte) (' ' + (temp % 95));
+ temp /= 95;
+ }
+ temp = rand.getLow8();
+ if (temp < 0) {
+ BigInteger bigTemp = makeBigInteger(temp);
+ recBuf[8] = (byte) (' ' + (bigTemp.mod(NINETY_FIVE).longValue()));
+ temp = bigTemp.divide(NINETY_FIVE).longValue();
+ } else {
+ recBuf[8] = (byte) (' ' + (temp % 95));
+ temp /= 95;
+ }
+ recBuf[9] = (byte)(' ' + (temp % 95));
+
+ /* add 2 bytes of "break" */
+ recBuf[10] = ' ';
+ recBuf[11] = ' ';
+
+ /* convert the 128-bit record number to 32 bits of ascii hexadecimal
+ * as the next 32 bytes of the record.
+ */
+ for (int i = 0; i < 32; i++) {
+ recBuf[12 + i] = (byte) recordNumber.getHexDigit(i);
+ }
+
+ /* add 2 bytes of "break" data */
+ recBuf[44] = ' ';
+ recBuf[45] = ' ';
+
+ /* add 52 bytes of filler based on low 48 bits of random number */
+ for(int i=0; i < 13; ++i) {
+ recBuf[46+i*4] = recBuf[47+i*4] = recBuf[48+i*4] = recBuf[49+i*4] =
+ (byte) rand.getHexDigit(19 + i);
+ }
+
+ /* add 2 bytes of "break" data */
+ recBuf[98] = '\r'; /* nice for Windows */
+ recBuf[99] = '\n';
+}
+
+
+ private static void usage() {
+ PrintStream out = System.out;
+ out.println("usage: gensort [-a] [-c] [-bSTARTING_REC_NUM] NUM_RECS FILE_NAME");
+ out.println("-a Generate ascii records required for PennySort or JouleSort.");
+ out.println(" These records are also an alternative input for the other");
+ out.println(" sort benchmarks. Without this flag, binary records will be");
+ out.println(" generated that contain the highest density of randomness in");
+ out.println(" the 10-byte key.");
+ out.println( "-c Calculate the sum of the crc32 checksums of each of the");
+ out.println(" generated records and send it to standard error.");
+ out.println("-bN Set the beginning record generated to N. By default the");
+ out.println(" first record generated is record 0.");
+ out.println("NUM_RECS The number of sequential records to generate.");
+ out.println("FILE_NAME The name of the file to write the records to.\n");
+ out.println("Example 1 - to generate 1000000 ascii records starting at record 0 to");
+ out.println("the file named \"pennyinput\":");
+ out.println(" gensort -a 1000000 pennyinput\n");
+ out.println("Example 2 - to generate 1000 binary records beginning with record 2000");
+ out.println("to the file named \"partition2\":");
+ out.println(" gensort -b2000 1000 partition2");
+ System.exit(1);
+ }
+
+
+ public static void outputRecords(OutputStream out,
+ boolean useAscii,
+ Unsigned16 firstRecordNumber,
+ Unsigned16 recordsToGenerate,
+ Unsigned16 checksum
+ ) throws IOException {
+ byte[] row = new byte[100];
+ Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
+ Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
+ Checksum crc = new PureJavaCrc32();
+ Unsigned16 tmp = new Unsigned16();
+ lastRecordNumber.add(recordsToGenerate);
+ Unsigned16 ONE = new Unsigned16(1);
+ Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
+ while (!recordNumber.equals(lastRecordNumber)) {
+ Random16.nextRand(rand);
+ if (useAscii) {
+ generateAsciiRecord(row, rand, recordNumber);
+ } else {
+ generateRecord(row, rand, recordNumber);
+ }
+ if (checksum != null) {
+ crc.reset();
+ crc.update(row, 0, row.length);
+ tmp.set(crc.getValue());
+ checksum.add(tmp);
+ }
+ recordNumber.add(ONE);
+ out.write(row);
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ Unsigned16 startingRecord = new Unsigned16();
+ Unsigned16 numberOfRecords;
+ OutputStream out;
+ boolean useAscii = false;
+ Unsigned16 checksum = null;
+
+ int i;
+ for(i=0; i < args.length; ++i) {
+ String arg = args[i];
+ int argLength = arg.length();
+ if (argLength >= 1 && arg.charAt(0) == '-') {
+ if (argLength < 2) {
+ usage();
+ }
+ switch (arg.charAt(1)) {
+ case 'a':
+ useAscii = true;
+ break;
+ case 'b':
+ startingRecord = Unsigned16.fromDecimal(arg.substring(2));
+ break;
+ case 'c':
+ checksum = new Unsigned16();
+ break;
+ default:
+ usage();
+ }
+ } else {
+ break;
+ }
+ }
+ if (args.length - i != 2) {
+ usage();
+ }
+ numberOfRecords = Unsigned16.fromDecimal(args[i]);
+ out = new FileOutputStream(args[i+1]);
+
+ outputRecords(out, useAscii, startingRecord, numberOfRecords, checksum);
+ out.close();
+ if (checksum != null) {
+ System.out.println(checksum);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/Random16.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/Random16.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/Random16.java
new file mode 100644
index 0000000..31cbd48
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/Random16.java
@@ -0,0 +1,374 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.mapreduce.examples.terasort;
+
+/**
+ * This class implements a 128-bit linear congruential generator.
+ * Specifically, if X0 is the most recently issued 128-bit random
+ * number (or a seed of 0 if no random number has already been generated,
+ * the next number to be generated, X1, is equal to:
+ * X1 = (a * X0 + c) mod 2**128
+ * where a is 47026247687942121848144207491837523525
+ * or 0x2360ed051fc65da44385df649fccf645
+ * and c is 98910279301475397889117759788405497857
+ * or 0x4a696d47726179524950202020202001
+ * The coefficient "a" is suggested by:
+ * Pierre L'Ecuyer, "Tables of linear congruential generators of different
+ * sizes and good lattice structure", Mathematics of Computation, 68
+ * pp. 249 - 260 (1999)
+ * http://www.ams.org/mcom/1999-68-225/S0025-5718-99-00996-5/S0025-5718-99-00996-5.pdf
+ * The constant "c" meets the simple suggestion by the same reference that
+ * it be odd.
+ *
+ * There is also a facility for quickly advancing the state of the
+ * generator by a fixed number of steps - this facilitates parallel
+ * generation.
+ *
+ * This is based on 1.0 of rand16.c from Chris Nyberg
+ * <ch...@ordinal.com>.
+ */
+class Random16 {
+
+ /**
+ * The "Gen" array contain powers of 2 of the linear congruential generator.
+ * The index 0 struct contain the "a" coefficient and "c" constant for the
+ * generator. That is, the generator is:
+ * f(x) = (Gen[0].a * x + Gen[0].c) mod 2**128
+ *
+ * All structs after the first contain an "a" and "c" that
+ * comprise the square of the previous function.
+ *
+ * f**2(x) = (Gen[1].a * x + Gen[1].c) mod 2**128
+ * f**4(x) = (Gen[2].a * x + Gen[2].c) mod 2**128
+ * f**8(x) = (Gen[3].a * x + Gen[3].c) mod 2**128
+ * ...
+
+ */
+ private static class RandomConstant {
+ final Unsigned16 a;
+ final Unsigned16 c;
+ public RandomConstant(String left, String right) {
+ a = new Unsigned16(left);
+ c = new Unsigned16(right);
+ }
+ }
+
+ private static final RandomConstant[] genArray = new RandomConstant[]{
+ /* [ 0] */ new RandomConstant("2360ed051fc65da44385df649fccf645",
+ "4a696d47726179524950202020202001"),
+ /* [ 1] */ new RandomConstant("17bce35bdf69743c529ed9eb20e0ae99",
+ "95e0e48262b3edfe04479485c755b646"),
+ /* [ 2] */ new RandomConstant("f4dd417327db7a9bd194dfbe42d45771",
+ "882a02c315362b60765f100068b33a1c"),
+ /* [ 3] */ new RandomConstant("6347af777a7898f6d1a2d6f33505ffe1",
+ "5efc4abfaca23e8ca8edb1f2dfbf6478"),
+ /* [ 4] */ new RandomConstant("b6a4239f3b315f84f6ef6d3d288c03c1",
+ "f25bd15439d16af594c1b1bafa6239f0"),
+ /* [ 5] */ new RandomConstant("2c82901ad1cb0cd182b631ba6b261781",
+ "89ca67c29c9397d59c612596145db7e0"),
+ /* [ 6] */ new RandomConstant("dab03f988288676ee49e66c4d2746f01",
+ "8b6ae036713bd578a8093c8eae5c7fc0"),
+ /* [ 7] */ new RandomConstant("602167331d86cf5684fe009a6d09de01",
+ "98a2542fd23d0dbdff3b886cdb1d3f80"),
+ /* [ 8] */ new RandomConstant("61ecb5c24d95b058f04c80a23697bc01",
+ "954db923fdb7933e947cd1edcecb7f00"),
+ /* [ 9] */ new RandomConstant("4a5c31e0654c28aa60474e83bf3f7801",
+ "00be4a36657c98cd204e8c8af7dafe00"),
+ /* [ 10] */ new RandomConstant("ae4f079d54fbece1478331d3c6bef001",
+ "991965329dccb28d581199ab18c5fc00"),
+ /* [ 11] */ new RandomConstant("101b8cb830c7cb927ff1ed50ae7de001",
+ "e1a8705b63ad5b8cd6c3d268d5cbf800"),
+ /* [ 12] */ new RandomConstant("f54a27fc056b00e7563f3505e0fbc001",
+ "2b657bbfd6ed9d632079e70c3c97f000"),
+ /* [ 13] */ new RandomConstant("df8a6fc1a833d201f98d719dd1f78001",
+ "59b60ee4c52fa49e9fe90682bd2fe000"),
+ /* [ 14] */ new RandomConstant("5480a5015f101a4ea7e3f183e3ef0001",
+ "cc099c88030679464fe86aae8a5fc000"),
+ /* [ 15] */ new RandomConstant("a498509e76e5d7925f539c28c7de0001",
+ "06b9abff9f9f33dd30362c0154bf8000"),
+ /* [ 16] */ new RandomConstant("0798a3d8b10dc72e60121cd58fbc0001",
+ "e296707121688d5a0260b293a97f0000"),
+ /* [ 17] */ new RandomConstant("1647d1e78ec02e665fafcbbb1f780001",
+ "189ffc4701ff23cb8f8acf6b52fe0000"),
+ /* [ 18] */ new RandomConstant("a7c982285e72bf8c0c8ddfb63ef00001",
+ "5141110ab208fb9d61fb47e6a5fc0000"),
+ /* [ 19] */ new RandomConstant("3eb78ee8fb8c56dbc5d4e06c7de00001",
+ "3c97caa62540f2948d8d340d4bf80000"),
+ /* [ 20] */ new RandomConstant("72d03b6f4681f2f9fe8e44d8fbc00001",
+ "1b25cb9cfe5a0c963174f91a97f00000"),
+ /* [ 21] */ new RandomConstant("ea85f81e4f502c9bc8ae99b1f7800001",
+ "0c644570b4a487103c5436352fe00000"),
+ /* [ 22] */ new RandomConstant("629c320db08b00c6bfa57363ef000001",
+ "3d0589c28869472bde517c6a5fc00000"),
+ /* [ 23] */ new RandomConstant("c5c4b9ce268d074a386be6c7de000001",
+ "bc95e5ab36477e65534738d4bf800000"),
+ /* [ 24] */ new RandomConstant("f30bbbbed1596187555bcd8fbc000001",
+ "ddb02ff72a031c01011f71a97f000000"),
+ /* [ 25] */ new RandomConstant("4a1000fb26c9eeda3cc79b1f78000001",
+ "2561426086d9acdb6c82e352fe000000"),
+ /* [ 26] */ new RandomConstant("89fb5307f6bf8ce2c1cf363ef0000001",
+ "64a788e3c118ed1c8215c6a5fc000000"),
+ /* [ 27] */ new RandomConstant("830b7b3358a5d67ea49e6c7de0000001",
+ "e65ea321908627cfa86b8d4bf8000000"),
+ /* [ 28] */ new RandomConstant("fd8a51da91a69fe1cd3cd8fbc0000001",
+ "53d27225604d85f9e1d71a97f0000000"),
+ /* [ 29] */ new RandomConstant("901a48b642b90b55aa79b1f780000001",
+ "ca5ec7a3ed1fe55e07ae352fe0000000"),
+ /* [ 30] */ new RandomConstant("118cdefdf32144f394f363ef00000001",
+ "4daebb2e085330651f5c6a5fc0000000"),
+ /* [ 31] */ new RandomConstant("0a88c0a91cff430829e6c7de00000001",
+ "9d6f1a00a8f3f76e7eb8d4bf80000000"),
+ /* [ 32] */ new RandomConstant("433bef4314f16a9453cd8fbc00000001",
+ "158c62f2b31e496dfd71a97f00000000"),
+ /* [ 33] */ new RandomConstant("c294b02995ae6738a79b1f7800000001",
+ "290e84a2eb15fd1ffae352fe00000000"),
+ /* [ 34] */ new RandomConstant("913575e0da8b16b14f363ef000000001",
+ "e3dc1bfbe991a34ff5c6a5fc00000000"),
+ /* [ 35] */ new RandomConstant("2f61b9f871cf4e629e6c7de000000001",
+ "ddf540d020b9eadfeb8d4bf800000000"),
+ /* [ 36] */ new RandomConstant("78d26ccbd68320c53cd8fbc000000001",
+ "8ee4950177ce66bfd71a97f000000000"),
+ /* [ 37] */ new RandomConstant("8b7ebd037898518a79b1f78000000001",
+ "39e0f787c907117fae352fe000000000"),
+ /* [ 38] */ new RandomConstant("0b5507b61f78e314f363ef0000000001",
+ "659d2522f7b732ff5c6a5fc000000000"),
+ /* [ 39] */ new RandomConstant("4f884628f812c629e6c7de0000000001",
+ "9e8722938612a5feb8d4bf8000000000"),
+ /* [ 40] */ new RandomConstant("be896744d4a98c53cd8fbc0000000001",
+ "e941a65d66b64bfd71a97f0000000000"),
+ /* [ 41] */ new RandomConstant("daf63a553b6318a79b1f780000000001",
+ "7b50d19437b097fae352fe0000000000"),
+ /* [ 42] */ new RandomConstant("2d7a23d8bf06314f363ef00000000001",
+ "59d7b68e18712ff5c6a5fc0000000000"),
+ /* [ 43] */ new RandomConstant("392b046a9f0c629e6c7de00000000001",
+ "4087bab2d5225feb8d4bf80000000000"),
+ /* [ 44] */ new RandomConstant("eb30fbb9c218c53cd8fbc00000000001",
+ "b470abc03b44bfd71a97f00000000000"),
+ /* [ 45] */ new RandomConstant("b9cdc30594318a79b1f7800000000001",
+ "366630eaba897fae352fe00000000000"),
+ /* [ 46] */ new RandomConstant("014ab453686314f363ef000000000001",
+ "a2dfc77e8512ff5c6a5fc00000000000"),
+ /* [ 47] */ new RandomConstant("395221c7d0c629e6c7de000000000001",
+ "1e0d25a14a25feb8d4bf800000000000"),
+ /* [ 48] */ new RandomConstant("4d972813a18c53cd8fbc000000000001",
+ "9d50a5d3944bfd71a97f000000000000"),
+ /* [ 49] */ new RandomConstant("06f9e2374318a79b1f78000000000001",
+ "bf7ab5eb2897fae352fe000000000000"),
+ /* [ 50] */ new RandomConstant("bd220cae86314f363ef0000000000001",
+ "925b14e6512ff5c6a5fc000000000000"),
+ /* [ 51] */ new RandomConstant("36fd3a5d0c629e6c7de0000000000001",
+ "724cce0ca25feb8d4bf8000000000000"),
+ /* [ 52] */ new RandomConstant("60def8ba18c53cd8fbc0000000000001",
+ "1af42d1944bfd71a97f0000000000000"),
+ /* [ 53] */ new RandomConstant("8d500174318a79b1f780000000000001",
+ "0f529e32897fae352fe0000000000000"),
+ /* [ 54] */ new RandomConstant("48e842e86314f363ef00000000000001",
+ "844e4c6512ff5c6a5fc0000000000000"),
+ /* [ 55] */ new RandomConstant("4af185d0c629e6c7de00000000000001",
+ "9f40d8ca25feb8d4bf80000000000000"),
+ /* [ 56] */ new RandomConstant("7a670ba18c53cd8fbc00000000000001",
+ "9912b1944bfd71a97f00000000000000"),
+ /* [ 57] */ new RandomConstant("86de174318a79b1f7800000000000001",
+ "9c69632897fae352fe00000000000000"),
+ /* [ 58] */ new RandomConstant("55fc2e86314f363ef000000000000001",
+ "e1e2c6512ff5c6a5fc00000000000000"),
+ /* [ 59] */ new RandomConstant("ccf85d0c629e6c7de000000000000001",
+ "68058ca25feb8d4bf800000000000000"),
+ /* [ 60] */ new RandomConstant("1df0ba18c53cd8fbc000000000000001",
+ "610b1944bfd71a97f000000000000000"),
+ /* [ 61] */ new RandomConstant("4be174318a79b1f78000000000000001",
+ "061632897fae352fe000000000000000"),
+ /* [ 62] */ new RandomConstant("d7c2e86314f363ef0000000000000001",
+ "1c2c6512ff5c6a5fc000000000000000"),
+ /* [ 63] */ new RandomConstant("af85d0c629e6c7de0000000000000001",
+ "7858ca25feb8d4bf8000000000000000"),
+ /* [ 64] */ new RandomConstant("5f0ba18c53cd8fbc0000000000000001",
+ "f0b1944bfd71a97f0000000000000000"),
+ /* [ 65] */ new RandomConstant("be174318a79b1f780000000000000001",
+ "e1632897fae352fe0000000000000000"),
+ /* [ 66] */ new RandomConstant("7c2e86314f363ef00000000000000001",
+ "c2c6512ff5c6a5fc0000000000000000"),
+ /* [ 67] */ new RandomConstant("f85d0c629e6c7de00000000000000001",
+ "858ca25feb8d4bf80000000000000000"),
+ /* [ 68] */ new RandomConstant("f0ba18c53cd8fbc00000000000000001",
+ "0b1944bfd71a97f00000000000000000"),
+ /* [ 69] */ new RandomConstant("e174318a79b1f7800000000000000001",
+ "1632897fae352fe00000000000000000"),
+ /* [ 70] */ new RandomConstant("c2e86314f363ef000000000000000001",
+ "2c6512ff5c6a5fc00000000000000000"),
+ /* [ 71] */ new RandomConstant("85d0c629e6c7de000000000000000001",
+ "58ca25feb8d4bf800000000000000000"),
+ /* [ 72] */ new RandomConstant("0ba18c53cd8fbc000000000000000001",
+ "b1944bfd71a97f000000000000000000"),
+ /* [ 73] */ new RandomConstant("174318a79b1f78000000000000000001",
+ "632897fae352fe000000000000000000"),
+ /* [ 74] */ new RandomConstant("2e86314f363ef0000000000000000001",
+ "c6512ff5c6a5fc000000000000000000"),
+ /* [ 75] */ new RandomConstant("5d0c629e6c7de0000000000000000001",
+ "8ca25feb8d4bf8000000000000000000"),
+ /* [ 76] */ new RandomConstant("ba18c53cd8fbc0000000000000000001",
+ "1944bfd71a97f0000000000000000000"),
+ /* [ 77] */ new RandomConstant("74318a79b1f780000000000000000001",
+ "32897fae352fe0000000000000000000"),
+ /* [ 78] */ new RandomConstant("e86314f363ef00000000000000000001",
+ "6512ff5c6a5fc0000000000000000000"),
+ /* [ 79] */ new RandomConstant("d0c629e6c7de00000000000000000001",
+ "ca25feb8d4bf80000000000000000000"),
+ /* [ 80] */ new RandomConstant("a18c53cd8fbc00000000000000000001",
+ "944bfd71a97f00000000000000000000"),
+ /* [ 81] */ new RandomConstant("4318a79b1f7800000000000000000001",
+ "2897fae352fe00000000000000000000"),
+ /* [ 82] */ new RandomConstant("86314f363ef000000000000000000001",
+ "512ff5c6a5fc00000000000000000000"),
+ /* [ 83] */ new RandomConstant("0c629e6c7de000000000000000000001",
+ "a25feb8d4bf800000000000000000000"),
+ /* [ 84] */ new RandomConstant("18c53cd8fbc000000000000000000001",
+ "44bfd71a97f000000000000000000000"),
+ /* [ 85] */ new RandomConstant("318a79b1f78000000000000000000001",
+ "897fae352fe000000000000000000000"),
+ /* [ 86] */ new RandomConstant("6314f363ef0000000000000000000001",
+ "12ff5c6a5fc000000000000000000000"),
+ /* [ 87] */ new RandomConstant("c629e6c7de0000000000000000000001",
+ "25feb8d4bf8000000000000000000000"),
+ /* [ 88] */ new RandomConstant("8c53cd8fbc0000000000000000000001",
+ "4bfd71a97f0000000000000000000000"),
+ /* [ 89] */ new RandomConstant("18a79b1f780000000000000000000001",
+ "97fae352fe0000000000000000000000"),
+ /* [ 90] */ new RandomConstant("314f363ef00000000000000000000001",
+ "2ff5c6a5fc0000000000000000000000"),
+ /* [ 91] */ new RandomConstant("629e6c7de00000000000000000000001",
+ "5feb8d4bf80000000000000000000000"),
+ /* [ 92] */ new RandomConstant("c53cd8fbc00000000000000000000001",
+ "bfd71a97f00000000000000000000000"),
+ /* [ 93] */ new RandomConstant("8a79b1f7800000000000000000000001",
+ "7fae352fe00000000000000000000000"),
+ /* [ 94] */ new RandomConstant("14f363ef000000000000000000000001",
+ "ff5c6a5fc00000000000000000000000"),
+ /* [ 95] */ new RandomConstant("29e6c7de000000000000000000000001",
+ "feb8d4bf800000000000000000000000"),
+ /* [ 96] */ new RandomConstant("53cd8fbc000000000000000000000001",
+ "fd71a97f000000000000000000000000"),
+ /* [ 97] */ new RandomConstant("a79b1f78000000000000000000000001",
+ "fae352fe000000000000000000000000"),
+ /* [ 98] */ new RandomConstant("4f363ef0000000000000000000000001",
+ "f5c6a5fc000000000000000000000000"),
+ /* [ 99] */ new RandomConstant("9e6c7de0000000000000000000000001",
+ "eb8d4bf8000000000000000000000000"),
+ /* [100] */ new RandomConstant("3cd8fbc0000000000000000000000001",
+ "d71a97f0000000000000000000000000"),
+ /* [101] */ new RandomConstant("79b1f780000000000000000000000001",
+ "ae352fe0000000000000000000000000"),
+ /* [102] */ new RandomConstant("f363ef00000000000000000000000001",
+ "5c6a5fc0000000000000000000000000"),
+ /* [103] */ new RandomConstant("e6c7de00000000000000000000000001",
+ "b8d4bf80000000000000000000000000"),
+ /* [104] */ new RandomConstant("cd8fbc00000000000000000000000001",
+ "71a97f00000000000000000000000000"),
+ /* [105] */ new RandomConstant("9b1f7800000000000000000000000001",
+ "e352fe00000000000000000000000000"),
+ /* [106] */ new RandomConstant("363ef000000000000000000000000001",
+ "c6a5fc00000000000000000000000000"),
+ /* [107] */ new RandomConstant("6c7de000000000000000000000000001",
+ "8d4bf800000000000000000000000000"),
+ /* [108] */ new RandomConstant("d8fbc000000000000000000000000001",
+ "1a97f000000000000000000000000000"),
+ /* [109] */ new RandomConstant("b1f78000000000000000000000000001",
+ "352fe000000000000000000000000000"),
+ /* [110] */ new RandomConstant("63ef0000000000000000000000000001",
+ "6a5fc000000000000000000000000000"),
+ /* [111] */ new RandomConstant("c7de0000000000000000000000000001",
+ "d4bf8000000000000000000000000000"),
+ /* [112] */ new RandomConstant("8fbc0000000000000000000000000001",
+ "a97f0000000000000000000000000000"),
+ /* [113] */ new RandomConstant("1f780000000000000000000000000001",
+ "52fe0000000000000000000000000000"),
+ /* [114] */ new RandomConstant("3ef00000000000000000000000000001",
+ "a5fc0000000000000000000000000000"),
+ /* [115] */ new RandomConstant("7de00000000000000000000000000001",
+ "4bf80000000000000000000000000000"),
+ /* [116] */ new RandomConstant("fbc00000000000000000000000000001",
+ "97f00000000000000000000000000000"),
+ /* [117] */ new RandomConstant("f7800000000000000000000000000001",
+ "2fe00000000000000000000000000000"),
+ /* [118] */ new RandomConstant("ef000000000000000000000000000001",
+ "5fc00000000000000000000000000000"),
+ /* [119] */ new RandomConstant("de000000000000000000000000000001",
+ "bf800000000000000000000000000000"),
+ /* [120] */ new RandomConstant("bc000000000000000000000000000001",
+ "7f000000000000000000000000000000"),
+ /* [121] */ new RandomConstant("78000000000000000000000000000001",
+ "fe000000000000000000000000000000"),
+ /* [122] */ new RandomConstant("f0000000000000000000000000000001",
+ "fc000000000000000000000000000000"),
+ /* [123] */ new RandomConstant("e0000000000000000000000000000001",
+ "f8000000000000000000000000000000"),
+ /* [124] */ new RandomConstant("c0000000000000000000000000000001",
+ "f0000000000000000000000000000000"),
+ /* [125] */ new RandomConstant("80000000000000000000000000000001",
+ "e0000000000000000000000000000000"),
+ /* [126] */ new RandomConstant("00000000000000000000000000000001",
+ "c0000000000000000000000000000000"),
+ /* [127] */ new RandomConstant("00000000000000000000000000000001",
+ "80000000000000000000000000000000")};
+
+ /**
+ * generate the random number that is "advance" steps
+ * from an initial random number of 0. This is done by
+ * starting with 0, and then advancing the by the
+ * appropriate powers of 2 of the linear congruential
+ * generator.
+ */
+ public static Unsigned16 skipAhead(Unsigned16 advance) {
+ Unsigned16 result = new Unsigned16();
+ long bit_map;
+
+ bit_map = advance.getLow8();
+ for (int i = 0; bit_map != 0 && i < 64; i++) {
+ if ((bit_map & (1L << i)) != 0) {
+ /* advance random number by f**(2**i) (x)
+ */
+ result.multiply(genArray[i].a);
+ result.add(genArray[i].c);
+ bit_map &= ~(1L << i);
+ }
+ }
+ bit_map = advance.getHigh8();
+ for (int i = 0; bit_map != 0 && i < 64; i++)
+ {
+ if ((bit_map & (1L << i)) != 0) {
+ /* advance random number by f**(2**(i + 64)) (x)
+ */
+ result.multiply(genArray[i+64].a);
+ result.add(genArray[i+64].c);
+ bit_map &= ~(1L << i);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Generate the next 16 byte random number.
+ */
+ public static void nextRand(Unsigned16 rand) {
+ /* advance the random number forward once using the linear congruential
+ * generator, and then return the new random number
+ */
+ rand.multiply(genArray[0].a);
+ rand.add(genArray[0].c);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraChecksum.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraChecksum.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraChecksum.java
new file mode 100644
index 0000000..a5b408b
--- /dev/null
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/terasort/TeraChecksum.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.mapreduce.examples.terasort;
+
+import java.io.IOException;
+import java.util.zip.Checksum;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.PureJavaCrc32;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+public class TeraChecksum extends Configured implements Tool {
+ static class ChecksumMapper
+ extends Mapper<Text, Text, NullWritable, Unsigned16> {
+ private Unsigned16 checksum = new Unsigned16();
+ private Unsigned16 sum = new Unsigned16();
+ private Checksum crc32 = new PureJavaCrc32();
+
+ public void map(Text key, Text value,
+ Context context) throws IOException {
+ crc32.reset();
+ crc32.update(key.getBytes(), 0, key.getLength());
+ crc32.update(value.getBytes(), 0, value.getLength());
+ checksum.set(crc32.getValue());
+ sum.add(checksum);
+ }
+
+ public void cleanup(Context context)
+ throws IOException, InterruptedException {
+ context.write(NullWritable.get(), sum);
+ }
+ }
+
+ static class ChecksumReducer
+ extends Reducer<NullWritable, Unsigned16, NullWritable, Unsigned16> {
+
+ public void reduce(NullWritable key, Iterable<Unsigned16> values,
+ Context context) throws IOException, InterruptedException {
+ Unsigned16 sum = new Unsigned16();
+ for (Unsigned16 val : values) {
+ sum.add(val);
+ }
+ context.write(key, sum);
+ }
+ }
+
+ private static void usage() throws IOException {
+ System.err.println("terasum <out-dir> <report-dir>");
+ }
+
+ public int run(String[] args) throws Exception {
+ Job job = Job.getInstance(getConf());
+ if (args.length != 2) {
+ usage();
+ return 2;
+ }
+ TeraInputFormat.setInputPaths(job, new Path(args[0]));
+ FileOutputFormat.setOutputPath(job, new Path(args[1]));
+ job.setJobName("TeraSum");
+ job.setJarByClass(TeraChecksum.class);
+ job.setMapperClass(ChecksumMapper.class);
+ job.setReducerClass(ChecksumReducer.class);
+ job.setOutputKeyClass(NullWritable.class);
+ job.setOutputValueClass(Unsigned16.class);
+ // force a single reducer
+ job.setNumReduceTasks(1);
+ job.setInputFormatClass(TeraInputFormat.class);
+ return job.waitForCompletion(true) ? 0 : 1;
+ }
+
+ /**
+ * @param args
+ */
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(), new TeraChecksum(), args);
+ System.exit(res);
+ }
+
+}
[09/10] TEZ-1055. Rename tez-mapreduce-examples to tez-examples
(Hitesh Shah via bikas)
Posted by bi...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
deleted file mode 100644
index 35a7c76..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
+++ /dev/null
@@ -1,259 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.tez.client.TezClient;
-import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.Edge;
-import org.apache.tez.dag.api.ProcessorDescriptor;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.api.Vertex;
-import org.apache.tez.dag.api.VertexManagerPluginDescriptor;
-import org.apache.tez.dag.api.client.DAGClient;
-import org.apache.tez.dag.api.client.DAGStatus;
-import org.apache.tez.dag.library.vertexmanager.InputReadyVertexManager;
-import org.apache.tez.runtime.api.ObjectRegistry;
-import org.apache.tez.runtime.api.ProcessorContext;
-import org.apache.tez.runtime.library.api.KeyValueReader;
-import org.apache.tez.runtime.library.api.KeyValueWriter;
-import org.apache.tez.runtime.library.conf.UnorderedUnpartitionedKVEdgeConfigurer;
-import org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput;
-import org.apache.tez.runtime.library.processor.SimpleProcessor;
-
-import com.google.common.base.Preconditions;
-
-public class BroadcastAndOneToOneExample extends Configured implements Tool {
- public static class InputProcessor extends SimpleProcessor {
- Text word = new Text();
-
- public InputProcessor(ProcessorContext context) {
- super(context);
- }
-
- @Override
- public void run() throws Exception {
- Preconditions.checkArgument(getOutputs().size() == 1);
- OnFileUnorderedKVOutput output = (OnFileUnorderedKVOutput) getOutputs().values().iterator()
- .next();
- KeyValueWriter kvWriter = (KeyValueWriter) output.getWriter();
- kvWriter.write(word, new IntWritable(getContext().getTaskIndex()));
- byte[] userPayload = getContext().getUserPayload().getPayload();
- if (userPayload != null) {
- boolean doLocalityCheck = userPayload[0] > 0 ? true : false;
- if (doLocalityCheck) {
- ObjectRegistry objectRegistry = getContext().getObjectRegistry();
- String entry = String.valueOf(getContext().getTaskIndex());
- objectRegistry.cacheForDAG(entry, entry);
- }
- }
- }
- }
-
- public static class OneToOneProcessor extends SimpleProcessor {
- Text word = new Text();
-
- public OneToOneProcessor(ProcessorContext context) {
- super(context);
- }
-
- @Override
- public void run() throws Exception {
- Preconditions.checkArgument(inputs.size() == 2);
-
- KeyValueReader inputKvReader = (KeyValueReader) getInputs().get("Input").getReader();
- KeyValueReader broadcastKvReader = (KeyValueReader) getInputs().get("Broadcast").getReader();
- int sum = 0;
- while (broadcastKvReader.next()) {
- sum += ((IntWritable) broadcastKvReader.getCurrentValue()).get();
- }
- while (inputKvReader.next()) {
- sum += ((IntWritable) inputKvReader.getCurrentValue()).get();
- }
- boolean doLocalityCheck = getContext().getUserPayload().getPayload()[0] > 0 ? true : false;
- int broadcastSum = getContext().getUserPayload().getPayload()[1];
- int expectedSum = broadcastSum + getContext().getTaskIndex();
- System.out.println("Index: " + getContext().getTaskIndex() +
- " sum: " + sum + " expectedSum: " + expectedSum + " broadcastSum: " + broadcastSum);
- Preconditions.checkState((sum == expectedSum), "Sum = " + sum);
-
- if (doLocalityCheck) {
- ObjectRegistry objectRegistry = getContext().getObjectRegistry();
- String index = (String) objectRegistry.get(String.valueOf(getContext().getTaskIndex()));
- if (index == null || Integer.valueOf(index).intValue() != getContext().getTaskIndex()) {
- String msg = "Did not find expected local producer "
- + getContext().getTaskIndex() + " in the same JVM";
- System.out.println(msg);
- throw new TezUncheckedException(msg);
- }
- }
- }
-
- }
-
- private DAG createDAG(FileSystem fs, TezConfiguration tezConf,
- Path stagingDir, boolean doLocalityCheck) throws IOException, YarnException {
-
- int numBroadcastTasks = 2;
- int numOneToOneTasks = 3;
- if (doLocalityCheck) {
- YarnClient yarnClient = YarnClient.createYarnClient();
- yarnClient.init(tezConf);
- yarnClient.start();
- int numNMs = yarnClient.getNodeReports(NodeState.RUNNING).size();
- yarnClient.stop();
- // create enough 1-1 tasks to run in parallel
- numOneToOneTasks = numNMs - numBroadcastTasks - 1;// 1 AM
- if (numOneToOneTasks < 1) {
- numOneToOneTasks = 1;
- }
- }
- byte[] procByte = {(byte) (doLocalityCheck ? 1 : 0), 1};
- UserPayload procPayload = new UserPayload(procByte);
-
- System.out.println("Using " + numOneToOneTasks + " 1-1 tasks");
-
- Vertex broadcastVertex = new Vertex("Broadcast", new ProcessorDescriptor(
- InputProcessor.class.getName()), numBroadcastTasks);
-
- Vertex inputVertex = new Vertex("Input", new ProcessorDescriptor(
- InputProcessor.class.getName()).setUserPayload(procPayload), numOneToOneTasks);
-
- Vertex oneToOneVertex = new Vertex("OneToOne",
- new ProcessorDescriptor(
- OneToOneProcessor.class.getName()).setUserPayload(procPayload));
- oneToOneVertex.setVertexManagerPlugin(
- new VertexManagerPluginDescriptor(InputReadyVertexManager.class.getName()));
-
- UnorderedUnpartitionedKVEdgeConfigurer edgeConf = UnorderedUnpartitionedKVEdgeConfigurer
- .newBuilder(Text.class.getName(), IntWritable.class.getName()).build();
-
- DAG dag = new DAG("BroadcastAndOneToOneExample");
- dag.addVertex(inputVertex)
- .addVertex(broadcastVertex)
- .addVertex(oneToOneVertex)
- .addEdge(
- new Edge(inputVertex, oneToOneVertex, edgeConf.createDefaultOneToOneEdgeProperty()))
- .addEdge(
- new Edge(broadcastVertex, oneToOneVertex,
- edgeConf.createDefaultBroadcastEdgeProperty()));
- return dag;
- }
-
- public boolean run(Configuration conf, boolean doLocalityCheck) throws Exception {
- System.out.println("Running BroadcastAndOneToOneExample");
- // conf and UGI
- TezConfiguration tezConf;
- if (conf != null) {
- tezConf = new TezConfiguration(conf);
- } else {
- tezConf = new TezConfiguration();
- }
- tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true);
- UserGroupInformation.setConfiguration(tezConf);
- String user = UserGroupInformation.getCurrentUser().getShortUserName();
-
- // staging dir
- FileSystem fs = FileSystem.get(tezConf);
- String stagingDirStr = Path.SEPARATOR + "user" + Path.SEPARATOR
- + user + Path.SEPARATOR+ ".staging" + Path.SEPARATOR
- + Path.SEPARATOR + Long.toString(System.currentTimeMillis());
- Path stagingDir = new Path(stagingDirStr);
- tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr);
- stagingDir = fs.makeQualified(stagingDir);
-
- // No need to add jar containing this class as assumed to be part of
- // the tez jars.
-
- // TEZ-674 Obtain tokens based on the Input / Output paths. For now assuming staging dir
- // is the same filesystem as the one used for Input/Output.
- TezClient tezSession = null;
- // needs session or else TaskScheduler does not hold onto containers
- tezSession = new TezClient("broadcastAndOneToOneExample", tezConf);
- tezSession.start();
-
- DAGClient dagClient = null;
-
- try {
- DAG dag = createDAG(fs, tezConf, stagingDir, doLocalityCheck);
-
- tezSession.waitTillReady();
- dagClient = tezSession.submitDAG(dag);
-
- // monitoring
- DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(null);
- if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
- System.out.println("DAG diagnostics: " + dagStatus.getDiagnostics());
- return false;
- }
- return true;
- } finally {
- fs.delete(stagingDir, true);
- tezSession.stop();
- }
- }
-
- @Override
- public int run(String[] args) throws Exception {
- boolean doLocalityCheck = true;
- if (args.length == 1) {
- if (args[0].equals(skipLocalityCheck)) {
- doLocalityCheck = false;
- } else {
- printUsage();
- throw new TezException("Invalid command line");
- }
- } else if (args.length > 1) {
- printUsage();
- throw new TezException("Invalid command line");
- }
- boolean status = run(getConf(), doLocalityCheck);
- return status ? 0 : 1;
- }
-
- private static void printUsage() {
- System.err.println("broadcastAndOneToOneExample " + skipLocalityCheck);
- ToolRunner.printGenericCommandUsage(System.err);
- }
-
- static String skipLocalityCheck = "-skipLocalityCheck";
-
- public static void main(String[] args) throws Exception {
- Configuration conf = new Configuration();
- BroadcastAndOneToOneExample job = new BroadcastAndOneToOneExample();
- int status = ToolRunner.run(conf, job, args);
- System.exit(status);
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
deleted file mode 100644
index f4f617a..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples;
-
-import java.io.IOException;
-import java.text.DecimalFormat;
-import java.util.EnumSet;
-import java.util.Set;
-
-import org.apache.hadoop.util.ProgramDriver;
-import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.client.DAGClient;
-import org.apache.tez.dag.api.client.DAGStatus;
-import org.apache.tez.dag.api.client.Progress;
-import org.apache.tez.dag.api.client.StatusGetOpts;
-import org.apache.tez.dag.api.client.VertexStatus;
-import org.apache.tez.examples.OrderedWordCount;
-import org.apache.tez.examples.SimpleSessionExample;
-import org.apache.tez.examples.WordCount;
-import org.apache.tez.mapreduce.examples.terasort.TeraGen;
-import org.apache.tez.mapreduce.examples.terasort.TeraSort;
-import org.apache.tez.mapreduce.examples.terasort.TeraValidate;
-
-/**
- * A description of an example program based on its class and a
- * human-readable description.
- */
-public class ExampleDriver {
-
- private static final DecimalFormat formatter = new DecimalFormat("###.##%");
-
- public static void main(String argv[]){
- int exitCode = -1;
- ProgramDriver pgd = new ProgramDriver();
- try {
- pgd.addClass("wordcount", WordCount.class,
- "A map/reduce program that counts the words in the input files.");
- pgd.addClass("mapredwordcount", MapredWordCount.class,
- "A map/reduce program that counts the words in the input files"
- + " using the mapred apis.");
- pgd.addClass("randomwriter", RandomWriter.class,
- "A map/reduce program that writes 10GB of random data per node.");
- pgd.addClass("randomtextwriter", RandomTextWriter.class,
- "A map/reduce program that writes 10GB of random textual data per node.");
- pgd.addClass("sort", Sort.class,
- "A map/reduce program that sorts the data written by the random"
- + " writer.");
- pgd.addClass("secondarysort", SecondarySort.class,
- "An example defining a secondary sort to the reduce.");
- pgd.addClass("join", Join.class,
- "A job that effects a join over sorted, equally partitioned"
- + " datasets");
- pgd.addClass("teragen", TeraGen.class,
- "Generate data for the terasort");
- pgd.addClass("terasort", TeraSort.class,
- "Run the terasort");
- pgd.addClass("teravalidate", TeraValidate.class,
- "Checking results of terasort");
- pgd.addClass("groupbyorderbymrrtest", GroupByOrderByMRRTest.class,
- "A map-reduce-reduce program that does groupby-order by. Takes input"
- + " containing employee_name department name per line of input"
- + " and generates count of employees per department and"
- + " sorted on employee count");
- pgd.addClass("mrrsleep", MRRSleepJob.class,
- "MRR Sleep Job");
- pgd.addClass("orderedwordcount", OrderedWordCount.class,
- "Word Count with words sorted on frequency");
- pgd.addClass("simplesessionexample", SimpleSessionExample.class,
- "Example to run multiple dags in a session");
- pgd.addClass("testorderedwordcount", TestOrderedWordCount.class,
- "Word Count with words sorted on frequency");
- pgd.addClass("unionexample", UnionExample.class,
- "Union example");
- pgd.addClass("broadcastAndOneToOneExample", BroadcastAndOneToOneExample.class,
- "BroadcastAndOneToOneExample example");
- pgd.addClass("filterLinesByWord", FilterLinesByWord.class,
- "Filters lines by the specified word using broadcast edge");
- pgd.addClass("filterLinesByWordOneToOne", FilterLinesByWordOneToOne.class,
- "Filters lines by the specified word using OneToOne edge");
- pgd.addClass("intersect", IntersectExample.class,
- "Identify all occurences of lines in file1 which also occur in file2");
- pgd.addClass("intersectdatagen", IntersectDataGen.class,
- "Generate data to run the intersect example");
- pgd.addClass("intersectvalidate", IntersectValidate.class,
- "Validate data generated by intersect and intersectdatagen");
- exitCode = pgd.run(argv);
- }
- catch(Throwable e){
- e.printStackTrace();
- }
-
- System.exit(exitCode);
- }
-
- public static void printDAGStatus(DAGClient dagClient, String[] vertexNames)
- throws IOException, TezException {
- printDAGStatus(dagClient, vertexNames, false, false);
- }
-
- public static void printDAGStatus(DAGClient dagClient, String[] vertexNames,
- boolean displayDAGCounters, boolean displayVertexCounters)
- throws IOException, TezException {
- Set<StatusGetOpts> opts = EnumSet.of(StatusGetOpts.GET_COUNTERS);
- DAGStatus dagStatus = dagClient.getDAGStatus(
- (displayDAGCounters ? opts : null));
- Progress progress = dagStatus.getDAGProgress();
- double vProgressFloat = 0.0f;
- if (progress != null) {
- System.out.println("");
- System.out.println("DAG: State: "
- + dagStatus.getState()
- + " Progress: "
- + (progress.getTotalTaskCount() < 0 ? formatter.format(0.0f) :
- formatter.format((double)(progress.getSucceededTaskCount())
- /progress.getTotalTaskCount())));
- for (String vertexName : vertexNames) {
- VertexStatus vStatus = dagClient.getVertexStatus(vertexName,
- (displayVertexCounters ? opts : null));
- if (vStatus == null) {
- System.out.println("Could not retrieve status for vertex: "
- + vertexName);
- continue;
- }
- Progress vProgress = vStatus.getProgress();
- if (vProgress != null) {
- vProgressFloat = 0.0f;
- if (vProgress.getTotalTaskCount() == 0) {
- vProgressFloat = 1.0f;
- } else if (vProgress.getTotalTaskCount() > 0) {
- vProgressFloat = (double)vProgress.getSucceededTaskCount()
- /vProgress.getTotalTaskCount();
- }
- System.out.println("VertexStatus:"
- + " VertexName: "
- + (vertexName.equals("ivertex1") ? "intermediate-reducer"
- : vertexName)
- + " Progress: " + formatter.format(vProgressFloat));
- }
- if (displayVertexCounters) {
- TezCounters counters = vStatus.getVertexCounters();
- if (counters != null) {
- System.out.println("Vertex Counters for " + vertexName + ": "
- + counters);
- }
- }
- }
- }
- if (displayDAGCounters) {
- TezCounters counters = dagStatus.getDAGCounters();
- if (counters != null) {
- System.out.println("DAG Counters: " + counters);
- }
- }
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
deleted file mode 100644
index 02a3202..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.UUID;
-
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.TextInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.security.TokenCache;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.util.ClassUtil;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.tez.client.TezClientUtils;
-import org.apache.tez.client.TezClient;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.DataSinkDescriptor;
-import org.apache.tez.dag.api.DataSourceDescriptor;
-import org.apache.tez.dag.api.Edge;
-import org.apache.tez.dag.api.OutputCommitterDescriptor;
-import org.apache.tez.dag.api.OutputDescriptor;
-import org.apache.tez.dag.api.ProcessorDescriptor;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.api.Vertex;
-import org.apache.tez.dag.api.client.DAGClient;
-import org.apache.tez.dag.api.client.DAGStatus;
-import org.apache.tez.dag.api.client.StatusGetOpts;
-import org.apache.tez.mapreduce.committer.MROutputCommitter;
-import org.apache.tez.mapreduce.examples.helpers.SplitsInClientOptionParser;
-import org.apache.tez.mapreduce.hadoop.MRInputHelpers;
-import org.apache.tez.mapreduce.input.MRInputLegacy;
-import org.apache.tez.mapreduce.output.MROutput;
-import org.apache.tez.processor.FilterByWordInputProcessor;
-import org.apache.tez.processor.FilterByWordOutputProcessor;
-import org.apache.tez.runtime.library.conf.UnorderedUnpartitionedKVEdgeConfigurer;
-
-import com.google.common.collect.Sets;
-
-public class FilterLinesByWord extends Configured implements Tool {
-
- private static Log LOG = LogFactory.getLog(FilterLinesByWord.class);
-
- public static final String FILTER_PARAM_NAME = "tez.runtime.examples.filterbyword.word";
-
- private boolean exitOnCompletion = false;
-
- public FilterLinesByWord(boolean exitOnCompletion) {
- this.exitOnCompletion = exitOnCompletion;
- }
-
- private static void printUsage() {
- System.err.println("Usage filtelinesrbyword <in> <out> <filter_word> [-generateSplitsInClient true/<false>]");
- ToolRunner.printGenericCommandUsage(System.err);
- }
-
- @Override
- public int run(String[] args) throws Exception {
- Configuration conf = getConf();
- String [] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
- Credentials credentials = new Credentials();
-
- boolean generateSplitsInClient = false;
-
- SplitsInClientOptionParser splitCmdLineParser = new SplitsInClientOptionParser();
- try {
- generateSplitsInClient = splitCmdLineParser.parse(otherArgs, false);
- otherArgs = splitCmdLineParser.getRemainingArgs();
- } catch (ParseException e1) {
- System.err.println("Invalid options");
- printUsage();
- return 2;
- }
-
- if (otherArgs.length != 3) {
- printUsage();
- return 2;
- }
-
- String inputPath = otherArgs[0];
- String outputPath = otherArgs[1];
- String filterWord = otherArgs[2];
-
- FileSystem fs = FileSystem.get(conf);
- if (fs.exists(new Path(outputPath))) {
- System.err.println("Output directory : " + outputPath + " already exists");
- return 2;
- }
-
- TezConfiguration tezConf = new TezConfiguration(conf);
-
- fs.getWorkingDirectory();
- Path stagingDir = new Path(fs.getWorkingDirectory(), UUID.randomUUID().toString());
- tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());
- TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
-
- String jarPath = ClassUtil.findContainingJar(FilterLinesByWord.class);
- if (jarPath == null) {
- throw new TezUncheckedException("Could not find any jar containing"
- + FilterLinesByWord.class.getName() + " in the classpath");
- }
-
- Path remoteJarPath = fs.makeQualified(new Path(stagingDir, "dag_job.jar"));
- fs.copyFromLocalFile(new Path(jarPath), remoteJarPath);
- FileStatus remoteJarStatus = fs.getFileStatus(remoteJarPath);
- TokenCache.obtainTokensForNamenodes(credentials, new Path[]{remoteJarPath}, conf);
-
- Map<String, LocalResource> commonLocalResources = new TreeMap<String, LocalResource>();
- LocalResource dagJarLocalRsrc = LocalResource.newInstance(
- ConverterUtils.getYarnUrlFromPath(remoteJarPath),
- LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
- remoteJarStatus.getLen(), remoteJarStatus.getModificationTime());
- commonLocalResources.put("dag_job.jar", dagJarLocalRsrc);
-
-
-
- TezClient tezSession = new TezClient("FilterLinesByWordSession", tezConf,
- commonLocalResources, credentials);
- tezSession.start(); // Why do I need to start the TezSession.
-
- Configuration stage1Conf = new JobConf(conf);
- stage1Conf.set(FILTER_PARAM_NAME, filterWord);
-
- Configuration stage2Conf = new JobConf(conf);
- stage2Conf.set(FileOutputFormat.OUTDIR, outputPath);
- stage2Conf.setBoolean("mapred.mapper.new-api", false);
-
- UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf);
- // Setup stage1 Vertex
- Vertex stage1Vertex = new Vertex("stage1", new ProcessorDescriptor(
- FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload))
- .setTaskLocalFiles(commonLocalResources);
-
- DataSourceDescriptor dsd;
- if (generateSplitsInClient) {
- // TODO TEZ-1406. Dont' use MRInputLegacy
- stage1Conf.set(FileInputFormat.INPUT_DIR, inputPath);
- stage1Conf.setBoolean("mapred.mapper.new-api", false);
- dsd = MRInputHelpers.configureMRInputWithLegacySplitGeneration(stage1Conf, stagingDir, true);
- } else {
- dsd = MRInputLegacy.createConfigurer(stage1Conf, TextInputFormat.class, inputPath)
- .groupSplits(false).create();
- }
- stage1Vertex.addDataSource("MRInput", dsd);
-
- // Setup stage2 Vertex
- Vertex stage2Vertex = new Vertex("stage2", new ProcessorDescriptor(
- FilterByWordOutputProcessor.class.getName()).setUserPayload(
- TezUtils.createUserPayloadFromConf(stage2Conf)), 1);
- stage2Vertex.setTaskLocalFiles(commonLocalResources);
-
- // Configure the Output for stage2
- OutputDescriptor od = new OutputDescriptor(MROutput.class.getName())
- .setUserPayload(TezUtils.createUserPayloadFromConf(stage2Conf));
- OutputCommitterDescriptor ocd = new OutputCommitterDescriptor(MROutputCommitter.class.getName());
- stage2Vertex.addDataSink("MROutput", new DataSinkDescriptor(od, ocd, null));
-
- UnorderedUnpartitionedKVEdgeConfigurer edgeConf = UnorderedUnpartitionedKVEdgeConfigurer
- .newBuilder(Text.class.getName(), TextLongPair.class.getName()).build();
-
- DAG dag = new DAG("FilterLinesByWord");
- Edge edge = new Edge(stage1Vertex, stage2Vertex, edgeConf.createDefaultBroadcastEdgeProperty());
- dag.addVertex(stage1Vertex).addVertex(stage2Vertex).addEdge(edge);
-
- LOG.info("Submitting DAG to Tez Session");
- DAGClient dagClient = tezSession.submitDAG(dag);
- LOG.info("Submitted DAG to Tez Session");
-
- DAGStatus dagStatus = null;
- String[] vNames = { "stage1", "stage2" };
- try {
- while (true) {
- dagStatus = dagClient.getDAGStatus(null);
- if(dagStatus.getState() == DAGStatus.State.RUNNING ||
- dagStatus.getState() == DAGStatus.State.SUCCEEDED ||
- dagStatus.getState() == DAGStatus.State.FAILED ||
- dagStatus.getState() == DAGStatus.State.KILLED ||
- dagStatus.getState() == DAGStatus.State.ERROR) {
- break;
- }
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- // continue;
- }
- }
-
- while (dagStatus.getState() == DAGStatus.State.RUNNING) {
- try {
- ExampleDriver.printDAGStatus(dagClient, vNames);
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- // continue;
- }
- dagStatus = dagClient.getDAGStatus(null);
- } catch (TezException e) {
- LOG.fatal("Failed to get application progress. Exiting");
- return -1;
- }
- }
-
- dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
-
- } finally {
- fs.delete(stagingDir, true);
- tezSession.stop();
- }
-
- ExampleDriver.printDAGStatus(dagClient, vNames, true, true);
- LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
- return dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1;
- }
-
- public static void main(String[] args) throws Exception {
- FilterLinesByWord fl = new FilterLinesByWord(true);
- int status = ToolRunner.run(new Configuration(), fl, args);
- if (fl.exitOnCompletion) {
- System.exit(status);
- }
- }
-
- public static class TextLongPair implements Writable {
-
- private Text text;
- private LongWritable longWritable;
-
- public TextLongPair() {
- }
-
- public TextLongPair(Text text, LongWritable longWritable) {
- this.text = text;
- this.longWritable = longWritable;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- this.text.write(out);
- this.longWritable.write(out);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- this.text = new Text();
- this.longWritable = new LongWritable();
- text.readFields(in);
- longWritable.readFields(in);
- }
-
- @Override
- public String toString() {
- return text.toString() + "\t" + longWritable.get();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
deleted file mode 100644
index b413993..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples;
-
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.UUID;
-
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.TextInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.util.ClassUtil;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.tez.client.TezClientUtils;
-import org.apache.tez.client.TezClient;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.DataSinkDescriptor;
-import org.apache.tez.dag.api.DataSourceDescriptor;
-import org.apache.tez.dag.api.Edge;
-import org.apache.tez.dag.api.OutputCommitterDescriptor;
-import org.apache.tez.dag.api.OutputDescriptor;
-import org.apache.tez.dag.api.ProcessorDescriptor;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.api.Vertex;
-import org.apache.tez.dag.api.client.DAGClient;
-import org.apache.tez.dag.api.client.DAGStatus;
-import org.apache.tez.mapreduce.committer.MROutputCommitter;
-import org.apache.tez.mapreduce.examples.FilterLinesByWord.TextLongPair;
-import org.apache.tez.mapreduce.examples.helpers.SplitsInClientOptionParser;
-import org.apache.tez.mapreduce.hadoop.MRInputHelpers;
-import org.apache.tez.mapreduce.input.MRInputLegacy;
-import org.apache.tez.mapreduce.output.MROutput;
-import org.apache.tez.processor.FilterByWordInputProcessor;
-import org.apache.tez.processor.FilterByWordOutputProcessor;
-import org.apache.tez.runtime.library.conf.UnorderedUnpartitionedKVEdgeConfigurer;
-
-public class FilterLinesByWordOneToOne extends Configured implements Tool {
-
- private static Log LOG = LogFactory.getLog(FilterLinesByWordOneToOne.class);
-
- public static final String FILTER_PARAM_NAME = "tez.runtime.examples.filterbyword.word";
-
- private static void printUsage() {
- System.err.println("Usage filterLinesByWordOneToOne <in> <out> <filter_word>"
- + " [-generateSplitsInClient true/<false>]");
- ToolRunner.printGenericCommandUsage(System.err);
- }
-
- public static void main(String[] args) throws Exception {
- Configuration conf = new Configuration();
- String[] otherArgs = new GenericOptionsParser(conf, args)
- .getRemainingArgs();
- int status = ToolRunner.run(conf, new FilterLinesByWordOneToOne(),
- otherArgs);
- System.exit(status);
- }
-
- @Override
- public int run(String[] otherArgs) throws Exception {
- boolean generateSplitsInClient = false;
- SplitsInClientOptionParser splitCmdLineParser = new SplitsInClientOptionParser();
- try {
- generateSplitsInClient = splitCmdLineParser.parse(otherArgs, false);
- otherArgs = splitCmdLineParser.getRemainingArgs();
- } catch (ParseException e1) {
- System.err.println("Invalid options");
- printUsage();
- return 2;
- }
-
- if (otherArgs.length != 3) {
- printUsage();
- return 2;
- }
-
- String inputPath = otherArgs[0];
- String outputPath = otherArgs[1];
- String filterWord = otherArgs[2];
-
- Configuration conf = getConf();
- FileSystem fs = FileSystem.get(conf);
- if (fs.exists(new Path(outputPath))) {
- System.err.println("Output directory : " + outputPath + " already exists");
- return 2;
- }
-
- TezConfiguration tezConf = new TezConfiguration(conf);
-
- fs.getWorkingDirectory();
- Path stagingDir = new Path(fs.getWorkingDirectory(), UUID.randomUUID().toString());
- tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());
- TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
-
- String jarPath = ClassUtil.findContainingJar(FilterLinesByWordOneToOne.class);
- if (jarPath == null) {
- throw new TezUncheckedException("Could not find any jar containing"
- + FilterLinesByWordOneToOne.class.getName() + " in the classpath");
- }
-
- Path remoteJarPath = fs.makeQualified(new Path(stagingDir, "dag_job.jar"));
- fs.copyFromLocalFile(new Path(jarPath), remoteJarPath);
- FileStatus remoteJarStatus = fs.getFileStatus(remoteJarPath);
-
- Map<String, LocalResource> commonLocalResources = new TreeMap<String, LocalResource>();
- LocalResource dagJarLocalRsrc = LocalResource.newInstance(
- ConverterUtils.getYarnUrlFromPath(remoteJarPath),
- LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
- remoteJarStatus.getLen(), remoteJarStatus.getModificationTime());
- commonLocalResources.put("dag_job.jar", dagJarLocalRsrc);
-
-
-
- TezClient tezSession = new TezClient("FilterLinesByWordSession", tezConf,
- commonLocalResources, null);
- tezSession.start(); // Why do I need to start the TezSession.
-
- Configuration stage1Conf = new JobConf(conf);
- stage1Conf.set(FILTER_PARAM_NAME, filterWord);
-
- Configuration stage2Conf = new JobConf(conf);
-
- stage2Conf.set(FileOutputFormat.OUTDIR, outputPath);
- stage2Conf.setBoolean("mapred.mapper.new-api", false);
-
- UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf);
- // Setup stage1 Vertex
- Vertex stage1Vertex = new Vertex("stage1", new ProcessorDescriptor(
- FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload))
- .setTaskLocalFiles(commonLocalResources);
-
- DataSourceDescriptor dsd;
- if (generateSplitsInClient) {
- // TODO TEZ-1406. Dont' use MRInputLegacy
- stage1Conf.set(FileInputFormat.INPUT_DIR, inputPath);
- stage1Conf.setBoolean("mapred.mapper.new-api", false);
- dsd = MRInputHelpers.configureMRInputWithLegacySplitGeneration(stage1Conf, stagingDir, true);
- } else {
- dsd = MRInputLegacy.createConfigurer(stage1Conf, TextInputFormat.class, inputPath)
- .groupSplits(false).create();
- }
- stage1Vertex.addDataSource("MRInput", dsd);
-
- // Setup stage2 Vertex
- Vertex stage2Vertex = new Vertex("stage2", new ProcessorDescriptor(
- FilterByWordOutputProcessor.class.getName()).setUserPayload(TezUtils
- .createUserPayloadFromConf(stage2Conf)), dsd.getNumberOfShards());
- stage2Vertex.setTaskLocalFiles(commonLocalResources);
-
- // Configure the Output for stage2
- stage2Vertex.addDataSink(
- "MROutput",
- new DataSinkDescriptor(new OutputDescriptor(MROutput.class.getName())
- .setUserPayload(TezUtils.createUserPayloadFromConf(stage2Conf)),
- new OutputCommitterDescriptor(MROutputCommitter.class.getName()), null));
-
- UnorderedUnpartitionedKVEdgeConfigurer edgeConf = UnorderedUnpartitionedKVEdgeConfigurer
- .newBuilder(Text.class.getName(), TextLongPair.class.getName()).build();
-
- DAG dag = new DAG("FilterLinesByWord");
- Edge edge = new Edge(stage1Vertex, stage2Vertex, edgeConf.createDefaultOneToOneEdgeProperty());
- dag.addVertex(stage1Vertex).addVertex(stage2Vertex).addEdge(edge);
-
- LOG.info("Submitting DAG to Tez Session");
- DAGClient dagClient = tezSession.submitDAG(dag);
- LOG.info("Submitted DAG to Tez Session");
-
- DAGStatus dagStatus = null;
- String[] vNames = { "stage1", "stage2" };
- try {
- while (true) {
- dagStatus = dagClient.getDAGStatus(null);
- if(dagStatus.getState() == DAGStatus.State.RUNNING ||
- dagStatus.getState() == DAGStatus.State.SUCCEEDED ||
- dagStatus.getState() == DAGStatus.State.FAILED ||
- dagStatus.getState() == DAGStatus.State.KILLED ||
- dagStatus.getState() == DAGStatus.State.ERROR) {
- break;
- }
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- // continue;
- }
- }
-
- while (dagStatus.getState() == DAGStatus.State.RUNNING) {
- try {
- ExampleDriver.printDAGStatus(dagClient, vNames);
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- // continue;
- }
- dagStatus = dagClient.getDAGStatus(null);
- } catch (TezException e) {
- LOG.fatal("Failed to get application progress. Exiting");
- return -1;
- }
- }
- } finally {
- fs.delete(stagingDir, true);
- tezSession.stop();
- }
-
- ExampleDriver.printDAGStatus(dagClient, vNames);
- LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
- return dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/GroupByOrderByMRRTest.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/GroupByOrderByMRRTest.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/GroupByOrderByMRRTest.java
deleted file mode 100644
index 939bea0..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/GroupByOrderByMRRTest.java
+++ /dev/null
@@ -1,247 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples;
-
-import java.io.IOException;
-import java.util.StringTokenizer;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.TypeConverter;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.tez.client.MRTezClient;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.client.DAGClient;
-import org.apache.tez.dag.api.client.DAGStatus;
-import org.apache.tez.mapreduce.hadoop.MRJobConfig;
-import org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil;
-
-/**
- * Simple example that does a GROUP BY ORDER BY in an MRR job
- * Consider a query such as
- * Select DeptName, COUNT(*) as cnt FROM EmployeeTable
- * GROUP BY DeptName ORDER BY cnt;
- *
- * i.e. List all departments with count of employees in each department
- * and ordered based on department's employee count.
- *
- * Requires an Input file containing 2 strings per line in format of
- * <EmployeeName> <DeptName>
- *
- * For example, use the following:
- *
- * #/bin/bash
- *
- * i=1000000
- * j=1000
- *
- * id=0
- * while [[ "$id" -ne "$i" ]]
- * do
- * id=`expr $id + 1`
- * deptId=`expr $RANDOM % $j + 1`
- * deptName=`echo "ibase=10;obase=16;$deptId" | bc`
- * echo "$id O$deptName"
- * done
- *
- */
-public class GroupByOrderByMRRTest extends Configured implements Tool {
-
- private static final Log LOG = LogFactory.getLog(GroupByOrderByMRRTest.class);
-
- /**
- * Mapper takes in a single line as input containing
- * employee name and department name and then
- * emits department name with count of 1
- */
- public static class MyMapper
- extends Mapper<Object, Text, Text, IntWritable> {
-
- private final static IntWritable one = new IntWritable(1);
- private final static Text word = new Text();
-
- public void map(Object key, Text value, Context context
- ) throws IOException, InterruptedException {
- StringTokenizer itr = new StringTokenizer(value.toString());
- String empName = "";
- String deptName = "";
- if (itr.hasMoreTokens()) {
- empName = itr.nextToken();
- if (itr.hasMoreTokens()) {
- deptName = itr.nextToken();
- }
- if (!empName.isEmpty()
- && !deptName.isEmpty()) {
- word.set(deptName);
- context.write(word, one);
- }
- }
- }
- }
-
- /**
- * Intermediate reducer aggregates the total count per department.
- * It takes department name and count as input and emits the final
- * count per department name.
- */
- public static class MyGroupByReducer
- extends Reducer<Text, IntWritable, IntWritable, Text> {
- private IntWritable result = new IntWritable();
-
- public void reduce(Text key, Iterable<IntWritable> values,
- Context context
- ) throws IOException, InterruptedException {
-
- int sum = 0;
- for (IntWritable val : values) {
- sum += val.get();
- }
- result.set(sum);
- context.write(result, key);
- }
- }
-
- /**
- * Shuffle ensures ordering based on count of employees per department
- * hence the final reducer is a no-op and just emits the department name
- * with the employee count per department.
- */
- public static class MyOrderByNoOpReducer
- extends Reducer<IntWritable, Text, Text, IntWritable> {
-
- public void reduce(IntWritable key, Iterable<Text> values,
- Context context
- ) throws IOException, InterruptedException {
- for (Text word : values) {
- context.write(word, key);
- }
- }
- }
-
- @Override
- public int run(String[] args) throws Exception {
- Configuration conf = getConf();
-
- // Configure intermediate reduces
- conf.setInt(MRJobConfig.MRR_INTERMEDIATE_STAGES, 1);
-
- // Set reducer class for intermediate reduce
- conf.setClass(MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1,
- "mapreduce.job.reduce.class"), MyGroupByReducer.class, Reducer.class);
- // Set reducer output key class
- conf.setClass(MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1,
- "mapreduce.map.output.key.class"), IntWritable.class, Object.class);
- // Set reducer output value class
- conf.setClass(MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1,
- "mapreduce.map.output.value.class"), Text.class, Object.class);
- conf.setInt(MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1,
- "mapreduce.job.reduces"), 2);
-
- String[] otherArgs = new GenericOptionsParser(conf, args).
- getRemainingArgs();
- if (otherArgs.length != 2) {
- System.err.println("Usage: groupbyorderbymrrtest <in> <out>");
- ToolRunner.printGenericCommandUsage(System.err);
- return 2;
- }
-
- @SuppressWarnings("deprecation")
- Job job = new Job(conf, "groupbyorderbymrrtest");
-
- job.setJarByClass(GroupByOrderByMRRTest.class);
-
- // Configure map
- job.setMapperClass(MyMapper.class);
- job.setMapOutputKeyClass(Text.class);
- job.setMapOutputValueClass(IntWritable.class);
-
- // Configure reduce
- job.setReducerClass(MyOrderByNoOpReducer.class);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
- job.setNumReduceTasks(1);
-
- FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
- FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
-
- job.submit();
- JobID jobId = job.getJobID();
- ApplicationId appId = TypeConverter.toYarn(jobId).getAppId();
-
- DAGClient dagClient = MRTezClient.getDAGClient(appId, new TezConfiguration(conf), null);
- DAGStatus dagStatus;
- String[] vNames = { "initialmap" , "ireduce1" , "finalreduce" };
- while (true) {
- dagStatus = dagClient.getDAGStatus(null);
- if(dagStatus.getState() == DAGStatus.State.RUNNING ||
- dagStatus.getState() == DAGStatus.State.SUCCEEDED ||
- dagStatus.getState() == DAGStatus.State.FAILED ||
- dagStatus.getState() == DAGStatus.State.KILLED ||
- dagStatus.getState() == DAGStatus.State.ERROR) {
- break;
- }
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- // continue;
- }
- }
-
- while (dagStatus.getState() == DAGStatus.State.RUNNING) {
- try {
- ExampleDriver.printDAGStatus(dagClient, vNames);
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- // continue;
- }
- dagStatus = dagClient.getDAGStatus(null);
- } catch (TezException e) {
- LOG.fatal("Failed to get application progress. Exiting");
- return -1;
- }
- }
-
- ExampleDriver.printDAGStatus(dagClient, vNames);
- LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
- return dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1;
- }
-
- public static void main(String[] args) throws Exception {
- Configuration configuration = new Configuration();
- GroupByOrderByMRRTest groupByOrderByMRRTest = new GroupByOrderByMRRTest();
- int status = ToolRunner.run(configuration, groupByOrderByMRRTest, args);
- System.exit(status);
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectDataGen.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectDataGen.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectDataGen.java
deleted file mode 100644
index 711240c..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectDataGen.java
+++ /dev/null
@@ -1,350 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.Random;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.tez.client.TezClient;
-import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.ProcessorDescriptor;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.api.Vertex;
-import org.apache.tez.dag.api.client.DAGClient;
-import org.apache.tez.dag.api.client.DAGStatus;
-import org.apache.tez.mapreduce.output.MROutput;
-import org.apache.tez.mapreduce.processor.SimpleMRProcessor;
-import org.apache.tez.runtime.api.ProcessorContext;
-import org.apache.tez.runtime.library.api.KeyValueWriter;
-
-import com.google.common.base.Preconditions;
-
-public class IntersectDataGen extends Configured implements Tool {
-
- private static final Log LOG = LogFactory.getLog(IntersectDataGen.class);
-
- private static final String STREAM_OUTPUT_NAME = "streamoutput";
- private static final String HASH_OUTPUT_NAME = "hashoutput";
- private static final String EXPECTED_OUTPUT_NAME = "expectedoutput";
-
- public static void main(String[] args) throws Exception {
- IntersectDataGen dataGen = new IntersectDataGen();
- int status = ToolRunner.run(new Configuration(), dataGen, args);
- System.exit(status);
- }
-
- private static void printUsage() {
- System.err
- .println("Usage: "
- + "intersectdatagen <outPath1> <path1Size> <outPath2> <path2Size> <expectedResultPath> <numTasks>");
- ;
- ToolRunner.printGenericCommandUsage(System.err);
- }
-
- @Override
- public int run(String[] args) throws Exception {
- Configuration conf = getConf();
- String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
- int result = validateArgs(otherArgs);
- if (result != 0) {
- return result;
- }
- return execute(otherArgs);
- }
-
- public int run(Configuration conf, String[] args, TezClient tezSession) throws Exception {
- setConf(conf);
- String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
- int result = validateArgs(otherArgs);
- if (result != 0) {
- return result;
- }
- return execute(otherArgs, tezSession);
- }
-
- private int validateArgs(String[] otherArgs) {
- if (otherArgs.length != 6) {
- printUsage();
- return 2;
- }
- return 0;
- }
-
- private int execute(String [] args) throws TezException, IOException, InterruptedException {
- TezConfiguration tezConf = new TezConfiguration(getConf());
- TezClient tezSession = null;
- try {
- tezSession = createTezSession(tezConf);
- return execute(args, tezConf, tezSession);
- } finally {
- if (tezSession != null) {
- tezSession.stop();
- }
- }
- }
-
- private int execute(String[] args, TezClient tezSession) throws IOException, TezException,
- InterruptedException {
- TezConfiguration tezConf = new TezConfiguration(getConf());
- return execute(args, tezConf, tezSession);
- }
-
- private TezClient createTezSession(TezConfiguration tezConf) throws TezException, IOException {
- TezClient tezSession = new TezClient("IntersectDataGenSession", tezConf);
- tezSession.start();
- return tezSession;
- }
-
- private int execute(String[] args, TezConfiguration tezConf, TezClient tezSession)
- throws IOException, TezException, InterruptedException {
- LOG.info("Running IntersectDataGen");
-
- UserGroupInformation.setConfiguration(tezConf);
-
- String outDir1 = args[0];
- long outDir1Size = Long.parseLong(args[1]);
- String outDir2 = args[2];
- long outDir2Size = Long.parseLong(args[3]);
- String expectedOutputDir = args[4];
- int numTasks = Integer.parseInt(args[5]);
-
- Path largeOutPath = null;
- Path smallOutPath = null;
- long largeOutSize = 0;
- long smallOutSize = 0;
-
- if (outDir1Size >= outDir2Size) {
- largeOutPath = new Path(outDir1);
- largeOutSize = outDir1Size;
- smallOutPath = new Path(outDir2);
- smallOutSize = outDir2Size;
- } else {
- largeOutPath = new Path(outDir2);
- largeOutSize = outDir2Size;
- smallOutPath = new Path(outDir1);
- smallOutSize = outDir1Size;
- }
-
- Path expectedOutputPath = new Path(expectedOutputDir);
-
- // Verify output path existence
- FileSystem fs = FileSystem.get(tezConf);
- int res = 0;
- res = checkOutputDirectory(fs, largeOutPath) + checkOutputDirectory(fs, smallOutPath)
- + checkOutputDirectory(fs, expectedOutputPath);
- if (res != 0) {
- return 3;
- }
-
- if (numTasks <= 0) {
- System.err.println("NumTasks must be > 0");
- return 4;
- }
-
- DAG dag = createDag(tezConf, largeOutPath, smallOutPath, expectedOutputPath, numTasks,
- largeOutSize, smallOutSize);
-
- tezSession.waitTillReady();
- DAGClient dagClient = tezSession.submitDAG(dag);
- DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(null);
- if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
- LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics());
- return -1;
- }
- return 0;
-
- }
-
- private DAG createDag(TezConfiguration tezConf, Path largeOutPath, Path smallOutPath,
- Path expectedOutputPath, int numTasks, long largeOutSize, long smallOutSize)
- throws IOException {
-
- long largeOutSizePerTask = largeOutSize / numTasks;
- long smallOutSizePerTask = smallOutSize / numTasks;
-
- DAG dag = new DAG("IntersectDataGen");
-
- Vertex genDataVertex = new Vertex("datagen", new ProcessorDescriptor(
- GenDataProcessor.class.getName()).setUserPayload(
- new UserPayload(GenDataProcessor.createConfiguration(largeOutSizePerTask, smallOutSizePerTask))), numTasks);
- genDataVertex.addDataSink(STREAM_OUTPUT_NAME,
- MROutput.createConfigurer(new Configuration(tezConf),
- TextOutputFormat.class, largeOutPath.toUri().toString()).create());
- genDataVertex.addDataSink(HASH_OUTPUT_NAME,
- MROutput.createConfigurer(new Configuration(tezConf),
- TextOutputFormat.class, smallOutPath.toUri().toString()).create());
- genDataVertex.addDataSink(EXPECTED_OUTPUT_NAME,
- MROutput.createConfigurer(new Configuration(tezConf),
- TextOutputFormat.class, expectedOutputPath.toUri().toString()).create());
-
- dag.addVertex(genDataVertex);
-
- return dag;
- }
-
- public static class GenDataProcessor extends SimpleMRProcessor {
-
- private static final Log LOG = LogFactory.getLog(GenDataProcessor.class);
-
- long streamOutputFileSize;
- long hashOutputFileSize;
- float overlapApprox = 0.2f;
-
- public GenDataProcessor(ProcessorContext context) {
- super(context);
- }
-
- public static byte[] createConfiguration(long streamOutputFileSize, long hashOutputFileSize)
- throws IOException {
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- DataOutputStream dos = new DataOutputStream(bos);
- dos.writeLong(streamOutputFileSize);
- dos.writeLong(hashOutputFileSize);
- dos.close();
- bos.close();
- return bos.toByteArray();
- }
-
- @Override
- public void initialize() throws Exception {
- byte[] payload = getContext().getUserPayload().getPayload();
- ByteArrayInputStream bis = new ByteArrayInputStream(payload);
- DataInputStream dis = new DataInputStream(bis);
- streamOutputFileSize = dis.readLong();
- hashOutputFileSize = dis.readLong();
- LOG.info("Initialized with largeFileTargetSize=" + streamOutputFileSize
- + ", smallFileTragetSize=" + hashOutputFileSize);
- dis.close();
- bis.close();
- }
-
- @Override
- public void run() throws Exception {
- Preconditions.checkState(getInputs().size() == 0);
- Preconditions.checkState(getOutputs().size() == 3);
-
- KeyValueWriter streamOutputWriter = (KeyValueWriter) getOutputs().get(STREAM_OUTPUT_NAME)
- .getWriter();
- KeyValueWriter hashOutputWriter = (KeyValueWriter) getOutputs().get(HASH_OUTPUT_NAME)
- .getWriter();
- KeyValueWriter expectedOutputWriter = (KeyValueWriter) getOutputs().get(EXPECTED_OUTPUT_NAME)
- .getWriter();
-
- float fileSizeFraction = hashOutputFileSize / (float) streamOutputFileSize;
- Preconditions.checkState(fileSizeFraction > 0.0f && fileSizeFraction <= 1.0f);
- int mod = 1;
- int extraKeysMod = 0;
- if (fileSizeFraction > overlapApprox) {
- // Common keys capped by overlap. Additional ones required in the hashFile.
- mod = (int) (1 / overlapApprox);
- extraKeysMod = (int) (1 / (fileSizeFraction - overlapApprox));
- } else {
- // All keys in hashFile must exist in stream file.
- mod = (int) (1 / fileSizeFraction);
- }
- LOG.info("Using mod=" + mod + ", extraKeysMod=" + extraKeysMod);
-
- long count = 0;
- long sizeLarge = 0;
- long sizeSmall = 0;
- long numLargeFileKeys = 0;
- long numSmallFileKeys = 0;
- long numExpectedKeys = 0;
- while (sizeLarge < streamOutputFileSize) {
- String str = createOverlapString(13, count);
- Text text = new Text(str);
- int size = text.getLength();
- streamOutputWriter.write(text, NullWritable.get());
- sizeLarge += size;
- numLargeFileKeys++;
- if (count % mod == 0) {
- hashOutputWriter.write(text, NullWritable.get());
- sizeSmall += size;
- numSmallFileKeys++;
- expectedOutputWriter.write(text, NullWritable.get());
- numExpectedKeys++;
- }
- if (extraKeysMod != 0 && count % extraKeysMod == 0) {
- String nStr = createNonOverlaptring(13, count);
- Text nText = new Text(nStr);
- hashOutputWriter.write(nText, NullWritable.get());
- sizeSmall += nText.getLength();
- numSmallFileKeys++;
- }
- count++;
- }
- LOG.info("OutputStats: " + "largeFileNumKeys=" + numLargeFileKeys + ", smallFileNumKeys="
- + numSmallFileKeys + ", expFileNumKeys=" + numExpectedKeys + ", largeFileSize="
- + sizeLarge + ", smallFileSize=" + sizeSmall);
- }
-
- private String createOverlapString(int size, long count) {
- StringBuilder sb = new StringBuilder();
- Random random = new Random();
- for (int i = 0; i < size; i++) {
- int r = Math.abs(random.nextInt()) % 26;
- // Random a-z followed by the count
- sb.append((char) (97 + r));
- }
- sb.append("_").append(getContext().getTaskIndex()).append("_").append(count);
- return sb.toString();
- }
-
- private String createNonOverlaptring(int size, long count) {
- StringBuilder sb = new StringBuilder();
- Random random = new Random();
- for (int i = 0; i < size; i++) {
- int r = Math.abs(random.nextInt()) % 26;
- // Random A-Z followed by the count
- sb.append((char) (65 + r));
- }
- sb.append("_").append(getContext().getTaskIndex()).append("_").append(count);
- return sb.toString();
- }
-
- }
-
- private int checkOutputDirectory(FileSystem fs, Path path) throws IOException {
- if (fs.exists(path)) {
- System.err.println("Output directory: " + path + " already exists");
- return 2;
- }
- return 0;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectExample.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectExample.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectExample.java
deleted file mode 100644
index da18014..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectExample.java
+++ /dev/null
@@ -1,281 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples;
-
-import java.io.IOException;
-import java.util.HashSet;
-
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.tez.client.TezClient;
-import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.Edge;
-import org.apache.tez.dag.api.ProcessorDescriptor;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.Vertex;
-import org.apache.tez.dag.api.client.DAGClient;
-import org.apache.tez.dag.api.client.DAGStatus;
-import org.apache.tez.mapreduce.input.MRInput;
-import org.apache.tez.mapreduce.output.MROutput;
-import org.apache.tez.mapreduce.processor.SimpleMRProcessor;
-import org.apache.tez.runtime.api.LogicalInput;
-import org.apache.tez.runtime.api.LogicalOutput;
-import org.apache.tez.runtime.api.Reader;
-import org.apache.tez.runtime.api.ProcessorContext;
-import org.apache.tez.runtime.library.api.KeyValueReader;
-import org.apache.tez.runtime.library.api.KeyValueWriter;
-import org.apache.tez.runtime.library.conf.UnorderedPartitionedKVEdgeConfigurer;
-import org.apache.tez.runtime.library.partitioner.HashPartitioner;
-import org.apache.tez.runtime.library.processor.SimpleProcessor;
-
-import com.google.common.base.Preconditions;
-
-public class IntersectExample extends Configured implements Tool {
-
- private static final Log LOG = LogFactory.getLog(IntersectExample.class);
-
- public static void main(String[] args) throws Exception {
- IntersectExample intersect = new IntersectExample();
- int status = ToolRunner.run(new Configuration(), intersect, args);
- System.exit(status);
- }
-
- private static void printUsage() {
- System.err.println("Usage: " + "intersect <file1> <file2> <numPartitions> <outPath>");
- ToolRunner.printGenericCommandUsage(System.err);
- }
-
- @Override
- public int run(String[] args) throws Exception {
- Configuration conf = getConf();
- String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
- int result = validateArgs(otherArgs);
- if (result != 0) {
- return result;
- }
- return execute(otherArgs);
- }
-
- public int run(Configuration conf, String[] args, TezClient tezSession) throws Exception {
- setConf(conf);
- String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
- int result = validateArgs(otherArgs);
- if (result != 0) {
- return result;
- }
- return execute(otherArgs, tezSession);
- }
-
- private int validateArgs(String[] otherArgs) {
- if (otherArgs.length != 4) {
- printUsage();
- return 2;
- }
- return 0;
- }
-
- private int execute(String[] args) throws TezException, IOException, InterruptedException {
- TezConfiguration tezConf = new TezConfiguration(getConf());
- TezClient tezSession = null;
- try {
- tezSession = createTezSession(tezConf);
- return execute(args, tezConf, tezSession);
- } finally {
- if (tezSession != null) {
- tezSession.stop();
- }
- }
- }
-
- private int execute(String[] args, TezClient tezSession) throws IOException, TezException,
- InterruptedException {
- TezConfiguration tezConf = new TezConfiguration(getConf());
- return execute(args, tezConf, tezSession);
- }
-
- private TezClient createTezSession(TezConfiguration tezConf) throws TezException, IOException {
- TezClient tezSession = new TezClient("IntersectExampleSession", tezConf);
- tezSession.start();
- return tezSession;
- }
-
- private int execute(String[] args, TezConfiguration tezConf, TezClient tezSession)
- throws IOException, TezException, InterruptedException {
- LOG.info("Running IntersectExample");
-
- UserGroupInformation.setConfiguration(tezConf);
-
- String streamInputDir = args[0];
- String hashInputDir = args[1];
- int numPartitions = Integer.parseInt(args[2]);
- String outputDir = args[3];
-
- Path streamInputPath = new Path(streamInputDir);
- Path hashInputPath = new Path(hashInputDir);
- Path outputPath = new Path(outputDir);
-
- // Verify output path existence
- FileSystem fs = FileSystem.get(tezConf);
- if (fs.exists(outputPath)) {
- System.err.println("Output directory: " + outputDir + " already exists");
- return 3;
- }
- if (numPartitions <= 0) {
- System.err.println("NumPartitions must be > 0");
- return 4;
- }
-
- DAG dag = createDag(tezConf, streamInputPath, hashInputPath, outputPath, numPartitions);
-
- tezSession.waitTillReady();
- DAGClient dagClient = tezSession.submitDAG(dag);
- DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(null);
- if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
- LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics());
- return -1;
- }
- return 0;
-
- }
-
- private DAG createDag(TezConfiguration tezConf, Path streamPath, Path hashPath, Path outPath,
- int numPartitions) throws IOException {
- DAG dag = new DAG("IntersectExample");
-
- // Configuration for intermediate output - shared by Vertex1 and Vertex2
- // This should only be setting selective keys from the underlying conf. Fix after there's a
- // better mechanism to configure the IOs.
-
- UnorderedPartitionedKVEdgeConfigurer edgeConf =
- UnorderedPartitionedKVEdgeConfigurer
- .newBuilder(Text.class.getName(), NullWritable.class.getName(),
- HashPartitioner.class.getName()).build();
-
- // Change the way resources are setup - no MRHelpers
- Vertex streamFileVertex = new Vertex("partitioner1", new ProcessorDescriptor(
- ForwardingProcessor.class.getName())).addDataSource(
- "streamfile",
- MRInput
- .createConfigurer(new Configuration(tezConf), TextInputFormat.class,
- streamPath.toUri().toString()).groupSplits(false).create());
-
- Vertex hashFileVertex = new Vertex("partitioner2", new ProcessorDescriptor(
- ForwardingProcessor.class.getName())).addDataSource(
- "hashfile",
- MRInput
- .createConfigurer(new Configuration(tezConf), TextInputFormat.class,
- hashPath.toUri().toString()).groupSplits(false).create());
-
- Vertex intersectVertex = new Vertex("intersect", new ProcessorDescriptor(
- IntersectProcessor.class.getName()), numPartitions).addDataSink("finalOutput",
- MROutput.createConfigurer(new Configuration(tezConf),
- TextOutputFormat.class, outPath.toUri().toString()).create());
-
- Edge e1 = new Edge(streamFileVertex, intersectVertex, edgeConf.createDefaultEdgeProperty());
-
- Edge e2 = new Edge(hashFileVertex, intersectVertex, edgeConf.createDefaultEdgeProperty());
-
- dag.addVertex(streamFileVertex).addVertex(hashFileVertex).addVertex(intersectVertex)
- .addEdge(e1).addEdge(e2);
- return dag;
- }
-
- // private void obtainTokens(Credentials credentials, Path... paths) throws IOException {
- // TokenCache.obtainTokensForNamenodes(credentials, paths, getConf());
- // }
-
- /**
- * Reads key-values from the source and forwards the value as the key for the output
- */
- public static class ForwardingProcessor extends SimpleProcessor {
- public ForwardingProcessor(ProcessorContext context) {
- super(context);
- }
-
- @Override
- public void run() throws Exception {
- Preconditions.checkState(getInputs().size() == 1);
- Preconditions.checkState(getOutputs().size() == 1);
- LogicalInput input = getInputs().values().iterator().next();
- Reader rawReader = input.getReader();
- Preconditions.checkState(rawReader instanceof KeyValueReader);
- LogicalOutput output = getOutputs().values().iterator().next();
-
- KeyValueReader reader = (KeyValueReader) rawReader;
- KeyValueWriter writer = (KeyValueWriter) output.getWriter();
-
- while (reader.next()) {
- Object val = reader.getCurrentValue();
- writer.write(val, NullWritable.get());
- }
- }
- }
-
- public static class IntersectProcessor extends SimpleMRProcessor {
-
- public IntersectProcessor(ProcessorContext context) {
- super(context);
- }
-
- @Override
- public void run() throws Exception {
- Preconditions.checkState(getInputs().size() == 2);
- Preconditions.checkState(getOutputs().size() == 1);
- LogicalInput streamInput = getInputs().get("partitioner1");
- LogicalInput hashInput = getInputs().get("partitioner2");
- Reader rawStreamReader = streamInput.getReader();
- Reader rawHashReader = hashInput.getReader();
- Preconditions.checkState(rawStreamReader instanceof KeyValueReader);
- Preconditions.checkState(rawHashReader instanceof KeyValueReader);
- LogicalOutput lo = getOutputs().values().iterator().next();
- Preconditions.checkState(lo instanceof MROutput);
- MROutput output = (MROutput) lo;
- KeyValueWriter writer = output.getWriter();
-
- KeyValueReader hashKvReader = (KeyValueReader) rawHashReader;
- Set<Text> keySet = new HashSet<Text>();
- while (hashKvReader.next()) {
- keySet.add(new Text((Text) hashKvReader.getCurrentKey()));
- }
-
- KeyValueReader streamKvReader = (KeyValueReader) rawStreamReader;
- while (streamKvReader.next()) {
- Text key = (Text) streamKvReader.getCurrentKey();
- if (keySet.contains(key)) {
- writer.write(key, NullWritable.get());
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectValidate.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectValidate.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectValidate.java
deleted file mode 100644
index c5328c7..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/IntersectValidate.java
+++ /dev/null
@@ -1,257 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.tez.client.TezClient;
-import org.apache.tez.common.counters.TezCounter;
-import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.Edge;
-import org.apache.tez.dag.api.ProcessorDescriptor;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.Vertex;
-import org.apache.tez.dag.api.client.DAGClient;
-import org.apache.tez.dag.api.client.DAGStatus;
-import org.apache.tez.dag.api.client.StatusGetOpts;
-import org.apache.tez.mapreduce.examples.IntersectExample.ForwardingProcessor;
-import org.apache.tez.mapreduce.input.MRInput;
-import org.apache.tez.runtime.api.LogicalInput;
-import org.apache.tez.runtime.api.Reader;
-import org.apache.tez.runtime.api.ProcessorContext;
-import org.apache.tez.runtime.library.api.KeyValuesReader;
-import org.apache.tez.runtime.library.conf.OrderedPartitionedKVEdgeConfigurer;
-import org.apache.tez.runtime.library.partitioner.HashPartitioner;
-import org.apache.tez.runtime.library.processor.SimpleProcessor;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Sets;
-
-public class IntersectValidate extends Configured implements Tool {
- private static final Log LOG = LogFactory.getLog(IntersectExample.class);
-
- private static final String LHS_INPUT_NAME = "lhsfile";
- private static final String RHS_INPUT_NAME = "rhsfile";
-
- private static final String COUNTER_GROUP_NAME = "INTERSECT_VALIDATE";
- private static final String MISSING_KEY_COUNTER_NAME = "MISSING_KEY_EXISTS";
-
- public static void main(String[] args) throws Exception {
- IntersectValidate validate = new IntersectValidate();
- int status = ToolRunner.run(new Configuration(), validate, args);
- System.exit(status);
- }
-
- private static void printUsage() {
- System.err.println("Usage: " + "intersectvalidate <path1> <path2>");
- ToolRunner.printGenericCommandUsage(System.err);
- }
-
- @Override
- public int run(String[] args) throws Exception {
- Configuration conf = getConf();
- String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
- int result = validateArgs(otherArgs);
- if (result != 0) {
- return result;
- }
- return execute(otherArgs);
- }
-
- public int run(Configuration conf, String[] args, TezClient tezSession) throws Exception {
- setConf(conf);
- String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
- int result = validateArgs(otherArgs);
- if (result != 0) {
- return result;
- }
- return execute(otherArgs, tezSession);
- }
-
- private int validateArgs(String[] otherArgs) {
- if (otherArgs.length != 3 && otherArgs.length != 2) {
- printUsage();
- return 2;
- }
- return 0;
- }
-
- private int execute(String[] args) throws TezException, IOException, InterruptedException {
- TezConfiguration tezConf = new TezConfiguration(getConf());
- TezClient tezSession = null;
- try {
- tezSession = createTezSession(tezConf);
- return execute(args, tezConf, tezSession);
- } finally {
- if (tezSession != null) {
- tezSession.stop();
- }
- }
- }
-
- private int execute(String[] args, TezClient tezSession) throws IOException, TezException,
- InterruptedException {
- TezConfiguration tezConf = new TezConfiguration(getConf());
- return execute(args, tezConf, tezSession);
- }
-
- private TezClient createTezSession(TezConfiguration tezConf) throws TezException, IOException {
- TezClient tezSession = new TezClient("IntersectValidateSession", tezConf);
- tezSession.start();
- return tezSession;
- }
-
- private int execute(String[] args, TezConfiguration tezConf, TezClient tezSession)
- throws IOException, TezException, InterruptedException {
- LOG.info("Running IntersectValidate");
- UserGroupInformation.setConfiguration(tezConf);
-
- String lhsDir = args[0];
- String rhsDir = args[1];
- int numPartitions = 1;
- if (args.length == 3) {
- numPartitions = Integer.parseInt(args[2]);
- }
-
- if (numPartitions <= 0) {
- System.err.println("NumPartitions must be > 0");
- return 4;
- }
-
- Path lhsPath = new Path(lhsDir);
- Path rhsPath = new Path(rhsDir);
-
- DAG dag = createDag(tezConf, lhsPath, rhsPath, numPartitions);
-
- tezSession.waitTillReady();
- DAGClient dagClient = tezSession.submitDAG(dag);
- DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(null);
- if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
- LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics());
- return -1;
- } else {
- dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
- TezCounter counter = dagStatus.getDAGCounters().findCounter(COUNTER_GROUP_NAME,
- MISSING_KEY_COUNTER_NAME);
- if (counter == null) {
- LOG.info("Unable to determing equality");
- return -2;
- } else {
- if (counter.getValue() != 0) {
- LOG.info("Validate failed. The two sides are not equivalent");
- return -3;
- } else {
- LOG.info("Validation successful. The two sides are equivalent");
- return 0;
- }
- }
- }
- }
-
- private DAG createDag(TezConfiguration tezConf, Path lhs, Path rhs, int numPartitions)
- throws IOException {
- DAG dag = new DAG("IntersectValidate");
-
- // Configuration for intermediate output - shared by Vertex1 and Vertex2
- // This should only be setting selective keys from the underlying conf. Fix after there's a
- // better mechanism to configure the IOs.
- OrderedPartitionedKVEdgeConfigurer edgeConf = OrderedPartitionedKVEdgeConfigurer
- .newBuilder(Text.class.getName(), NullWritable.class.getName(),
- HashPartitioner.class.getName()).build();
-
- Vertex lhsVertex = new Vertex(LHS_INPUT_NAME, new ProcessorDescriptor(
- ForwardingProcessor.class.getName())).addDataSource("lhs",
- MRInput
- .createConfigurer(new Configuration(tezConf), TextInputFormat.class,
- lhs.toUri().toString()).groupSplits(false).create());
-
- Vertex rhsVertex = new Vertex(RHS_INPUT_NAME, new ProcessorDescriptor(
- ForwardingProcessor.class.getName())).addDataSource("rhs",
- MRInput
- .createConfigurer(new Configuration(tezConf), TextInputFormat.class,
- rhs.toUri().toString()).groupSplits(false).create());
-
- Vertex intersectValidateVertex = new Vertex("intersectvalidate", new ProcessorDescriptor(
- IntersectValidateProcessor.class.getName()), numPartitions);
-
- Edge e1 = new Edge(lhsVertex, intersectValidateVertex, edgeConf.createDefaultEdgeProperty());
- Edge e2 = new Edge(rhsVertex, intersectValidateVertex, edgeConf.createDefaultEdgeProperty());
-
- dag.addVertex(lhsVertex).addVertex(rhsVertex).addVertex(intersectValidateVertex).addEdge(e1)
- .addEdge(e2);
- return dag;
- }
-
- public static class IntersectValidateProcessor extends SimpleProcessor {
-
- private static final Log LOG = LogFactory.getLog(IntersectValidateProcessor.class);
-
- public IntersectValidateProcessor(ProcessorContext context) {
- super(context);
- }
-
- @Override
- public void run() throws Exception {
- Preconditions.checkState(getInputs().size() == 2);
- Preconditions.checkState(getOutputs().size() == 0);
- LogicalInput lhsInput = getInputs().get(LHS_INPUT_NAME);
- LogicalInput rhsInput = getInputs().get(RHS_INPUT_NAME);
- Reader lhsReaderRaw = lhsInput.getReader();
- Reader rhsReaderRaw = rhsInput.getReader();
- Preconditions.checkState(lhsReaderRaw instanceof KeyValuesReader);
- Preconditions.checkState(rhsReaderRaw instanceof KeyValuesReader);
- KeyValuesReader lhsReader = (KeyValuesReader) lhsReaderRaw;
- KeyValuesReader rhsReader = (KeyValuesReader) rhsReaderRaw;
-
- TezCounter lhsMissingKeyCounter = getContext().getCounters().findCounter(COUNTER_GROUP_NAME,
- MISSING_KEY_COUNTER_NAME);
-
- while (lhsReader.next()) {
- if (rhsReader.next()) {
- if (!lhsReader.getCurrentKey().equals(rhsReader.getCurrentKey())) {
- LOG.info("MismatchedKeys: " + "lhs=" + lhsReader.getCurrentKey() + ", rhs=" + rhsReader.getCurrentKey());
- lhsMissingKeyCounter.increment(1);
- }
- } else {
- lhsMissingKeyCounter.increment(1);
- LOG.info("ExtraKey in lhs: " + lhsReader.getClass());
- break;
- }
- }
- if (rhsReader.next()) {
- lhsMissingKeyCounter.increment(1);
- LOG.info("ExtraKey in rhs: " + lhsReader.getClass());
- }
- }
- }
-
-}