You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by bi...@apache.org on 2014/08/16 02:54:56 UTC
[08/10] TEZ-1055. Rename tez-mapreduce-examples to tez-examples
(Hitesh Shah via bikas)
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/Join.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/Join.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/Join.java
deleted file mode 100644
index a3a65ec..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/Join.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples;
-
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.mapred.ClusterStatus;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.join.CompositeInputFormat;
-import org.apache.hadoop.mapreduce.lib.join.TupleWritable;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-/**
- * Given a set of sorted datasets keyed with the same class and yielding
- * equal partitions, it is possible to effect a join of those datasets
- * prior to the map. The example facilitates the same.
- *
- * To run: bin/hadoop jar build/hadoop-examples.jar join
- * [-r <i>reduces</i>]
- * [-inFormat <i>input format class</i>]
- * [-outFormat <i>output format class</i>]
- * [-outKey <i>output key class</i>]
- * [-outValue <i>output value class</i>]
- * [-joinOp <inner|outer|override>]
- * [<i>in-dir</i>]* <i>in-dir</i> <i>out-dir</i>
- */
-public class Join extends Configured implements Tool {
- public static final String REDUCES_PER_HOST = "mapreduce.join.reduces_per_host";
- static int printUsage() {
- System.out.println("join [-r <reduces>] " +
- "[-inFormat <input format class>] " +
- "[-outFormat <output format class>] " +
- "[-outKey <output key class>] " +
- "[-outValue <output value class>] " +
- "[-joinOp <inner|outer|override>] " +
- "[input]* <input> <output>");
- ToolRunner.printGenericCommandUsage(System.out);
- return 2;
- }
-
- /**
- * The main driver for sort program.
- * Invoke this method to submit the map/reduce job.
- * @throws IOException When there is communication problems with the
- * job tracker.
- */
- @SuppressWarnings("unchecked")
- public int run(String[] args) throws Exception {
- Configuration conf = getConf();
- JobClient client = new JobClient(conf);
- ClusterStatus cluster = client.getClusterStatus();
- int num_reduces = (int) (cluster.getMaxReduceTasks() * 0.9);
- String join_reduces = conf.get(REDUCES_PER_HOST);
- if (join_reduces != null) {
- num_reduces = cluster.getTaskTrackers() *
- Integer.parseInt(join_reduces);
- }
- Job job = new Job(conf);
- job.setJobName("join");
- job.setJarByClass(Sort.class);
-
- job.setMapperClass(Mapper.class);
- job.setReducerClass(Reducer.class);
-
- Class<? extends InputFormat> inputFormatClass =
- SequenceFileInputFormat.class;
- Class<? extends OutputFormat> outputFormatClass =
- SequenceFileOutputFormat.class;
- Class<? extends WritableComparable> outputKeyClass = BytesWritable.class;
- Class<? extends Writable> outputValueClass = TupleWritable.class;
- String op = "inner";
- List<String> otherArgs = new ArrayList<String>();
- for(int i=0; i < args.length; ++i) {
- try {
- if ("-r".equals(args[i])) {
- num_reduces = Integer.parseInt(args[++i]);
- } else if ("-inFormat".equals(args[i])) {
- inputFormatClass =
- Class.forName(args[++i]).asSubclass(InputFormat.class);
- } else if ("-outFormat".equals(args[i])) {
- outputFormatClass =
- Class.forName(args[++i]).asSubclass(OutputFormat.class);
- } else if ("-outKey".equals(args[i])) {
- outputKeyClass =
- Class.forName(args[++i]).asSubclass(WritableComparable.class);
- } else if ("-outValue".equals(args[i])) {
- outputValueClass =
- Class.forName(args[++i]).asSubclass(Writable.class);
- } else if ("-joinOp".equals(args[i])) {
- op = args[++i];
- } else {
- otherArgs.add(args[i]);
- }
- } catch (NumberFormatException except) {
- System.out.println("ERROR: Integer expected instead of " + args[i]);
- return printUsage();
- } catch (ArrayIndexOutOfBoundsException except) {
- System.out.println("ERROR: Required parameter missing from " +
- args[i-1]);
- return printUsage(); // exits
- }
- }
-
- // Set user-supplied (possibly default) job configs
- job.setNumReduceTasks(num_reduces);
-
- if (otherArgs.size() < 2) {
- System.out.println("ERROR: Wrong number of parameters: ");
- return printUsage();
- }
-
- FileOutputFormat.setOutputPath(job,
- new Path(otherArgs.remove(otherArgs.size() - 1)));
- List<Path> plist = new ArrayList<Path>(otherArgs.size());
- for (String s : otherArgs) {
- plist.add(new Path(s));
- }
-
- job.setInputFormatClass(CompositeInputFormat.class);
- job.getConfiguration().set(CompositeInputFormat.JOIN_EXPR,
- CompositeInputFormat.compose(op, inputFormatClass,
- plist.toArray(new Path[0])));
- job.setOutputFormatClass(outputFormatClass);
-
- job.setOutputKeyClass(outputKeyClass);
- job.setOutputValueClass(outputValueClass);
-
- Date startTime = new Date();
- System.out.println("Job started: " + startTime);
- int ret = job.waitForCompletion(true) ? 0 : 1 ;
- Date end_time = new Date();
- System.out.println("Job ended: " + end_time);
- System.out.println("The job took " +
- (end_time.getTime() - startTime.getTime()) /1000 + " seconds.");
- return ret;
- }
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new Join(), args);
- System.exit(res);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
deleted file mode 100644
index b22a4a8..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
+++ /dev/null
@@ -1,757 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.tez.mapreduce.examples;
-
-import java.io.IOException;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.google.common.collect.Maps;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Partitioner;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
-import org.apache.hadoop.mapreduce.security.TokenCache;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.util.ClassUtil;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.tez.client.TezClientUtils;
-import org.apache.tez.client.TezClient;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.DataSourceDescriptor;
-import org.apache.tez.dag.api.Edge;
-import org.apache.tez.dag.api.ProcessorDescriptor;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.api.Vertex;
-import org.apache.tez.dag.api.client.DAGClient;
-import org.apache.tez.dag.api.client.DAGStatus;
-import org.apache.tez.mapreduce.hadoop.MRHelpers;
-import org.apache.tez.mapreduce.hadoop.MRInputHelpers;
-import org.apache.tez.mapreduce.hadoop.MultiStageMRConfigUtil;
-import org.apache.tez.mapreduce.input.MRInputLegacy;
-import org.apache.tez.mapreduce.output.MROutputLegacy;
-import org.apache.tez.mapreduce.processor.map.MapProcessor;
-import org.apache.tez.mapreduce.processor.reduce.ReduceProcessor;
-import org.apache.tez.runtime.library.conf.OrderedPartitionedKVEdgeConfigurer;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.tez.runtime.library.partitioner.HashPartitioner;
-
-/**
- * Dummy class for testing MR framefork. Sleeps for a defined period
- * of time in mapper and reducer. Generates fake input for map / reduce
- * jobs. Note that generated number of input pairs is in the order
- * of <code>numMappers * mapSleepTime / 100</code>, so the job uses
- * some disk space.
- */
-public class MRRSleepJob extends Configured implements Tool {
-
- private static final Log LOG = LogFactory.getLog(MRRSleepJob.class);
-
- public static final String MAP_SLEEP_COUNT = "mrr.sleepjob.map.sleep.count";
- public static final String REDUCE_SLEEP_COUNT =
- "mrr.sleepjob.reduce.sleep.count";
- public static final String MAP_SLEEP_TIME = "mrr.sleepjob.map.sleep.time";
- public static final String REDUCE_SLEEP_TIME =
- "mrr.sleepjob.reduce.sleep.time";
- public static final String IREDUCE_SLEEP_COUNT =
- "mrr.sleepjob.ireduce.sleep.count";
- public static final String IREDUCE_SLEEP_TIME =
- "mrr.sleepjob.ireduce.sleep.time";
- public static final String IREDUCE_STAGES_COUNT =
- "mrr.sleepjob.ireduces.stages.count";
- public static final String IREDUCE_TASKS_COUNT =
- "mrr.sleepjob.ireduces.tasks.count";
-
- // Flags to inject failures
- public static final String MAP_THROW_ERROR = "mrr.sleepjob.map.throw.error";
- public static final String MAP_FATAL_ERROR = "mrr.sleepjob.map.fatal.error";
- public static final String MAP_ERROR_TASK_IDS =
- "mrr.sleepjob.map.error.task.ids";
-
- public static class MRRSleepJobPartitioner extends
- Partitioner<IntWritable, IntWritable> {
- public int getPartition(IntWritable k, IntWritable v, int numPartitions) {
- return k.get() % numPartitions;
- }
- }
-
- public static class EmptySplit extends InputSplit implements Writable {
- public void write(DataOutput out) throws IOException { }
- public void readFields(DataInput in) throws IOException { }
- public long getLength() { return 0L; }
- public String[] getLocations() { return new String[0]; }
- }
-
- public static class SleepInputFormat
- extends InputFormat<IntWritable,IntWritable> {
-
- public List<InputSplit> getSplits(JobContext jobContext) {
- List<InputSplit> ret = new ArrayList<InputSplit>();
- int numSplits = jobContext.getConfiguration().
- getInt(MRJobConfig.NUM_MAPS, 1);
- for (int i = 0; i < numSplits; ++i) {
- ret.add(new EmptySplit());
- }
- return ret;
- }
-
- public RecordReader<IntWritable,IntWritable> createRecordReader(
- InputSplit ignored, TaskAttemptContext taskContext)
- throws IOException {
- Configuration conf = taskContext.getConfiguration();
-
- final int count = conf.getInt(MAP_SLEEP_COUNT, 1);
- if (count < 0) {
- throw new IOException("Invalid map count: " + count);
- }
-
- int totalIReduces = conf.getInt(IREDUCE_STAGES_COUNT, 1);
-
- int reduceTasks = totalIReduces == 0?
- taskContext.getNumReduceTasks() :
- conf.getInt(IREDUCE_TASKS_COUNT, 1);
- int sleepCount = totalIReduces == 0?
- conf.getInt(REDUCE_SLEEP_COUNT,1) :
- conf.getInt(IREDUCE_SLEEP_COUNT,1);
- final int emitPerMapTask = sleepCount * reduceTasks;
-
- return new RecordReader<IntWritable,IntWritable>() {
- private int records = 0;
- private int emitCount = 0;
- private IntWritable key = null;
- private IntWritable value = null;
-
- public void initialize(InputSplit split, TaskAttemptContext context) {
- }
-
- public boolean nextKeyValue()
- throws IOException {
- if (count == 0) {
- return false;
- }
- key = new IntWritable();
- key.set(emitCount);
- int emit = emitPerMapTask / count;
- if ((emitPerMapTask) % count > records) {
- ++emit;
- }
- emitCount += emit;
- value = new IntWritable();
- value.set(emit);
- return records++ < count;
- }
- public IntWritable getCurrentKey() { return key; }
- public IntWritable getCurrentValue() { return value; }
- public void close() throws IOException { }
- public float getProgress() throws IOException {
- return count == 0 ? 100 : records / ((float)count);
- }
- };
- }
- }
-
- public static class SleepMapper
- extends Mapper<IntWritable, IntWritable, IntWritable, IntWritable> {
- private long mapSleepDuration = 100;
- private int mapSleepCount = 1;
- private int count = 0;
- private String vertexName;
- private boolean throwError = false;
- private boolean throwFatal = false;
- private boolean finalAttempt = false;
-
- protected void setup(Context context)
- throws IOException, InterruptedException {
- Configuration conf = context.getConfiguration();
- this.mapSleepCount =
- conf.getInt(MAP_SLEEP_COUNT, mapSleepCount);
- this.mapSleepDuration = mapSleepCount == 0 ? 0 :
- conf.getLong(MAP_SLEEP_TIME , 100) / mapSleepCount;
- vertexName = conf.get(
- org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);
-
- TaskAttemptID taId = context.getTaskAttemptID();
-
- String[] taskIds = conf.getStrings(MAP_ERROR_TASK_IDS);
- if (taId.getId()+1 >= context.getMaxMapAttempts()) {
- finalAttempt = true;
- }
- boolean found = false;
- if (taskIds != null) {
- if (taskIds.length == 1 && taskIds[0].equals("*")) {
- found = true;
- }
- if (!found) {
- for (String taskId : taskIds) {
- if (Integer.valueOf(taskId).intValue() ==
- taId.getTaskID().getId()) {
- found = true;
- break;
- }
- }
- }
- }
- if (found) {
- if (!finalAttempt) {
- throwError = conf.getBoolean(MAP_THROW_ERROR, false);
- }
- throwFatal = conf.getBoolean(MAP_FATAL_ERROR, false);
- }
- }
-
- public void map(IntWritable key, IntWritable value, Context context
- ) throws IOException, InterruptedException {
- //it is expected that every map processes mapSleepCount number of records.
- try {
- LOG.info("Reading in " + vertexName
- + " taskid " + context.getTaskAttemptID().getTaskID().getId()
- + " key " + key.get());
- LOG.info("Sleeping in InitialMap"
- + ", vertexName=" + vertexName
- + ", taskAttemptId=" + context.getTaskAttemptID()
- + ", mapSleepDuration=" + mapSleepDuration
- + ", mapSleepCount=" + mapSleepCount
- + ", sleepLeft="
- + (mapSleepDuration * (mapSleepCount - count)));
- context.setStatus("Sleeping... (" +
- (mapSleepDuration * (mapSleepCount - count)) + ") ms left");
- if ((mapSleepCount - count) > 0) {
- Thread.sleep(mapSleepDuration);
- }
- if (throwError || throwFatal) {
- throw new IOException("Throwing a simulated error from map");
- }
- }
- catch (InterruptedException ex) {
- throw (IOException)new IOException(
- "Interrupted while sleeping").initCause(ex);
- }
- ++count;
- // output reduceSleepCount * numReduce number of random values, so that
- // each reducer will get reduceSleepCount number of keys.
- int k = key.get();
- for (int i = 0; i < value.get(); ++i) {
- LOG.info("Writing in " + vertexName
- + " taskid " + context.getTaskAttemptID().getTaskID().getId()
- + " key " + (k+i) + " value 1");
- context.write(new IntWritable(k + i), new IntWritable(1));
- }
- }
- }
-
- public static class ISleepReducer
- extends Reducer<IntWritable, IntWritable, IntWritable, IntWritable> {
- private long iReduceSleepDuration = 100;
- private int iReduceSleepCount = 1;
- private int count = 0;
- private String vertexName;
-
- protected void setup(Context context)
- throws IOException, InterruptedException {
- Configuration conf = context.getConfiguration();
- this.iReduceSleepCount =
- conf.getInt(IREDUCE_SLEEP_COUNT, iReduceSleepCount);
- this.iReduceSleepDuration = iReduceSleepCount == 0 ? 0 :
- conf.getLong(IREDUCE_SLEEP_TIME , 100) / iReduceSleepCount;
- vertexName = conf.get(
- org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);
- }
-
- public void reduce(IntWritable key, Iterable<IntWritable> values,
- Context context)
- throws IOException, InterruptedException {
- try {
- LOG.info("Reading in " + vertexName
- + " taskid " + context.getTaskAttemptID().getTaskID().getId()
- + " key " + key.get());
-
- LOG.info("Sleeping in IntermediateReduce"
- + ", vertexName=" + vertexName
- + ", taskAttemptId=" + context.getTaskAttemptID()
- + ", iReduceSleepDuration=" + iReduceSleepDuration
- + ", iReduceSleepCount=" + iReduceSleepCount
- + ", sleepLeft="
- + (iReduceSleepDuration * (iReduceSleepCount - count)));
- context.setStatus("Sleeping... (" +
- (iReduceSleepDuration * (iReduceSleepCount - count)) + ") ms left");
- if ((iReduceSleepCount - count) > 0) {
- Thread.sleep(iReduceSleepDuration);
- }
- }
- catch (InterruptedException ex) {
- throw (IOException)new IOException(
- "Interrupted while sleeping").initCause(ex);
- }
- ++count;
- // output reduceSleepCount * numReduce number of random values, so that
- // each reducer will get reduceSleepCount number of keys.
- int k = key.get();
- for (IntWritable value : values) {
- for (int i = 0; i < value.get(); ++i) {
- LOG.info("Writing in " + vertexName
- + " taskid " + context.getTaskAttemptID().getTaskID().getId()
- + " key " + (k+i) + " value 1");
- context.write(new IntWritable(k + i), new IntWritable(1));
- }
- }
- }
- }
-
- public static class SleepReducer
- extends Reducer<IntWritable, IntWritable, NullWritable, NullWritable> {
- private long reduceSleepDuration = 100;
- private int reduceSleepCount = 1;
- private int count = 0;
- private String vertexName;
-
- protected void setup(Context context)
- throws IOException, InterruptedException {
- Configuration conf = context.getConfiguration();
- this.reduceSleepCount =
- conf.getInt(REDUCE_SLEEP_COUNT, reduceSleepCount);
- this.reduceSleepDuration = reduceSleepCount == 0 ? 0 :
- conf.getLong(REDUCE_SLEEP_TIME , 100) / reduceSleepCount;
- vertexName = conf.get(
- org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME);
- }
-
- public void reduce(IntWritable key, Iterable<IntWritable> values,
- Context context)
- throws IOException {
- try {
- LOG.info("Reading in " + vertexName
- + " taskid " + context.getTaskAttemptID().getTaskID().getId()
- + " key " + key.get());
- LOG.info("Sleeping in FinalReduce"
- + ", vertexName=" + vertexName
- + ", taskAttemptId=" + context.getTaskAttemptID()
- + ", reduceSleepDuration=" + reduceSleepDuration
- + ", reduceSleepCount=" + reduceSleepCount
- + ", sleepLeft="
- + (reduceSleepDuration * (reduceSleepCount - count)));
- context.setStatus("Sleeping... (" +
- (reduceSleepDuration * (reduceSleepCount - count)) + ") ms left");
- if ((reduceSleepCount - count) > 0) {
- Thread.sleep(reduceSleepDuration);
- }
- }
- catch (InterruptedException ex) {
- throw (IOException)new IOException(
- "Interrupted while sleeping").initCause(ex);
- }
- count++;
- }
- }
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new MRRSleepJob(), args);
- System.exit(res);
- }
-
- private Credentials credentials = new Credentials();
-
- public DAG createDAG(FileSystem remoteFs, Configuration conf, Path remoteStagingDir,
- int numMapper, int numReducer, int iReduceStagesCount,
- int numIReducer, long mapSleepTime, int mapSleepCount,
- long reduceSleepTime, int reduceSleepCount,
- long iReduceSleepTime, int iReduceSleepCount, boolean writeSplitsToDFS,
- boolean generateSplitsInAM)
- throws IOException, YarnException {
-
-
- Configuration mapStageConf = new JobConf(conf);
- mapStageConf.setInt(MRJobConfig.NUM_MAPS, numMapper);
- mapStageConf.setLong(MAP_SLEEP_TIME, mapSleepTime);
- mapStageConf.setLong(REDUCE_SLEEP_TIME, reduceSleepTime);
- mapStageConf.setLong(IREDUCE_SLEEP_TIME, iReduceSleepTime);
- mapStageConf.setInt(MAP_SLEEP_COUNT, mapSleepCount);
- mapStageConf.setInt(REDUCE_SLEEP_COUNT, reduceSleepCount);
- mapStageConf.setInt(IREDUCE_SLEEP_COUNT, iReduceSleepCount);
- mapStageConf.setInt(IREDUCE_STAGES_COUNT, iReduceStagesCount);
- mapStageConf.setInt(IREDUCE_TASKS_COUNT, numIReducer);
- mapStageConf.set(MRJobConfig.MAP_CLASS_ATTR, SleepMapper.class.getName());
- mapStageConf.set(MRJobConfig.INPUT_FORMAT_CLASS_ATTR,
- SleepInputFormat.class.getName());
- if (numIReducer == 0 && numReducer == 0) {
- mapStageConf.set(MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR,
- NullOutputFormat.class.getName());
- }
-
- MRHelpers.translateMRConfToTez(mapStageConf);
-
- Configuration[] intermediateReduceStageConfs = null;
- if (iReduceStagesCount > 0
- && numIReducer > 0) {
- intermediateReduceStageConfs = new JobConf[iReduceStagesCount];
- for (int i = 1; i <= iReduceStagesCount; ++i) {
- JobConf iReduceStageConf = new JobConf(conf);
- iReduceStageConf.setLong(MRRSleepJob.REDUCE_SLEEP_TIME, iReduceSleepTime);
- iReduceStageConf.setInt(MRRSleepJob.REDUCE_SLEEP_COUNT, iReduceSleepCount);
- iReduceStageConf.setInt(MRJobConfig.NUM_REDUCES, numIReducer);
- iReduceStageConf
- .set(MRJobConfig.REDUCE_CLASS_ATTR, ISleepReducer.class.getName());
- iReduceStageConf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS,
- IntWritable.class.getName());
- iReduceStageConf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS,
- IntWritable.class.getName());
- iReduceStageConf.set(MRJobConfig.PARTITIONER_CLASS_ATTR,
- MRRSleepJobPartitioner.class.getName());
-
-
- MRHelpers.translateMRConfToTez(iReduceStageConf);
- intermediateReduceStageConfs[i-1] = iReduceStageConf;
- }
- }
-
- Configuration finalReduceConf = null;
- if (numReducer > 0) {
- finalReduceConf = new JobConf(conf);
- finalReduceConf.setLong(MRRSleepJob.REDUCE_SLEEP_TIME, reduceSleepTime);
- finalReduceConf.setInt(MRRSleepJob.REDUCE_SLEEP_COUNT, reduceSleepCount);
- finalReduceConf.setInt(MRJobConfig.NUM_REDUCES, numReducer);
- finalReduceConf.set(MRJobConfig.REDUCE_CLASS_ATTR, SleepReducer.class.getName());
- finalReduceConf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS,
- IntWritable.class.getName());
- finalReduceConf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS,
- IntWritable.class.getName());
- finalReduceConf.set(MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR,
- NullOutputFormat.class.getName());
-
- MRHelpers.translateMRConfToTez(finalReduceConf);
- }
-
- MRHelpers.configureMRApiUsage(mapStageConf);
- if (iReduceStagesCount > 0
- && numIReducer > 0) {
- for (int i = 0; i < iReduceStagesCount; ++i) {
- MRHelpers.configureMRApiUsage(intermediateReduceStageConfs[i]);
- }
- }
- if (numReducer > 0) {
- MRHelpers.configureMRApiUsage(finalReduceConf);
- }
-
- DataSourceDescriptor dataSource = null;
- if (!generateSplitsInAM && writeSplitsToDFS) {
-
- LOG.info("Writing splits to DFS");
- dataSource = MRInputHelpers
- .configureMRInputWithLegacySplitGeneration(mapStageConf, remoteStagingDir, true);
- } else {
- dataSource = MRInputLegacy.createConfigurer(mapStageConf, SleepInputFormat.class)
- .generateSplitsInAM(generateSplitsInAM).create();
- }
-
- DAG dag = new DAG("MRRSleepJob");
- String jarPath = ClassUtil.findContainingJar(getClass());
- if (jarPath == null) {
- throw new TezUncheckedException("Could not find any jar containing"
- + " MRRSleepJob.class in the classpath");
- }
- Path remoteJarPath = remoteFs.makeQualified(
- new Path(remoteStagingDir, "dag_job.jar"));
- remoteFs.copyFromLocalFile(new Path(jarPath), remoteJarPath);
- FileStatus jarFileStatus = remoteFs.getFileStatus(remoteJarPath);
-
- TokenCache.obtainTokensForNamenodes(this.credentials, new Path[] { remoteJarPath },
- mapStageConf);
-
- Map<String, LocalResource> commonLocalResources =
- new HashMap<String, LocalResource>();
- LocalResource dagJarLocalRsrc = LocalResource.newInstance(
- ConverterUtils.getYarnUrlFromPath(remoteJarPath),
- LocalResourceType.FILE,
- LocalResourceVisibility.APPLICATION,
- jarFileStatus.getLen(),
- jarFileStatus.getModificationTime());
- commonLocalResources.put("dag_job.jar", dagJarLocalRsrc);
-
- List<Vertex> vertices = new ArrayList<Vertex>();
-
-
- UserPayload mapUserPayload = TezUtils.createUserPayloadFromConf(mapStageConf);
- int numTasks = generateSplitsInAM ? -1 : numMapper;
-
- Vertex mapVertex = new Vertex("map", new ProcessorDescriptor(
- MapProcessor.class.getName()).setUserPayload(mapUserPayload), numTasks)
- .setTaskLocalFiles(commonLocalResources);
- mapVertex.addDataSource("MRInput", dataSource);
- vertices.add(mapVertex);
-
- if (iReduceStagesCount > 0
- && numIReducer > 0) {
- for (int i = 0; i < iReduceStagesCount; ++i) {
- Configuration iconf =
- intermediateReduceStageConfs[i];
- UserPayload iReduceUserPayload = TezUtils.createUserPayloadFromConf(iconf);
- Vertex ivertex = new Vertex("ireduce" + (i+1),
- new ProcessorDescriptor(ReduceProcessor.class.getName()).
- setUserPayload(iReduceUserPayload), numIReducer);
- ivertex.setTaskLocalFiles(commonLocalResources);
- vertices.add(ivertex);
- }
- }
-
- Vertex finalReduceVertex = null;
- if (numReducer > 0) {
- UserPayload reducePayload = TezUtils.createUserPayloadFromConf(finalReduceConf);
- finalReduceVertex = new Vertex("reduce", new ProcessorDescriptor(
- ReduceProcessor.class.getName()).setUserPayload(reducePayload), numReducer);
- finalReduceVertex.setTaskLocalFiles(commonLocalResources);
- finalReduceVertex.addDataSink("MROutput", MROutputLegacy.createConfigurer(finalReduceConf,
- NullOutputFormat.class).create());
- vertices.add(finalReduceVertex);
- } else {
- // Map only job
- mapVertex.addDataSink("MROutput",
- MROutputLegacy.createConfigurer(mapStageConf, NullOutputFormat.class).create());
- }
-
-
- Map<String, String> partitionerConf = Maps.newHashMap();
- partitionerConf.put(MRJobConfig.PARTITIONER_CLASS_ATTR, MRRSleepJobPartitioner.class.getName());
- OrderedPartitionedKVEdgeConfigurer edgeConf = OrderedPartitionedKVEdgeConfigurer
- .newBuilder(IntWritable.class.getName(), IntWritable.class.getName(),
- HashPartitioner.class.getName(), partitionerConf).configureInput().useLegacyInput()
- .done().build();
-
- for (int i = 0; i < vertices.size(); ++i) {
- dag.addVertex(vertices.get(i));
- if (i != 0) {
- dag.addEdge(
- new Edge(vertices.get(i - 1), vertices.get(i), edgeConf.createDefaultEdgeProperty()));
- }
- }
-
- return dag;
- }
-
- @VisibleForTesting
- public Job createJob(int numMapper, int numReducer, int iReduceStagesCount,
- int numIReducer, long mapSleepTime, int mapSleepCount,
- long reduceSleepTime, int reduceSleepCount,
- long iReduceSleepTime, int iReduceSleepCount)
- throws IOException {
- Configuration conf = getConf();
- conf.setLong(MAP_SLEEP_TIME, mapSleepTime);
- conf.setLong(REDUCE_SLEEP_TIME, reduceSleepTime);
- conf.setLong(IREDUCE_SLEEP_TIME, iReduceSleepTime);
- conf.setInt(MAP_SLEEP_COUNT, mapSleepCount);
- conf.setInt(REDUCE_SLEEP_COUNT, reduceSleepCount);
- conf.setInt(IREDUCE_SLEEP_COUNT, iReduceSleepCount);
- conf.setInt(MRJobConfig.NUM_MAPS, numMapper);
- conf.setInt(IREDUCE_STAGES_COUNT, iReduceStagesCount);
- conf.setInt(IREDUCE_TASKS_COUNT, numIReducer);
-
- // Configure intermediate reduces
- conf.setInt(
- org.apache.tez.mapreduce.hadoop.MRJobConfig.MRR_INTERMEDIATE_STAGES,
- iReduceStagesCount);
- LOG.info("Running MRR with " + iReduceStagesCount + " IR stages");
-
- for (int i = 1; i <= iReduceStagesCount; ++i) {
- // Set reducer class for intermediate reduce
- conf.setClass(
- MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,
- "mapreduce.job.reduce.class"), ISleepReducer.class, Reducer.class);
- // Set reducer output key class
- conf.setClass(
- MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,
- "mapreduce.map.output.key.class"), IntWritable.class, Object.class);
- // Set reducer output value class
- conf.setClass(
- MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,
- "mapreduce.map.output.value.class"), IntWritable.class, Object.class);
- conf.setInt(
- MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i,
- "mapreduce.job.reduces"), numIReducer);
- }
-
- Job job = Job.getInstance(conf, "sleep");
- job.setNumReduceTasks(numReducer);
- job.setJarByClass(MRRSleepJob.class);
- job.setNumReduceTasks(numReducer);
- job.setMapperClass(SleepMapper.class);
- job.setMapOutputKeyClass(IntWritable.class);
- job.setMapOutputValueClass(IntWritable.class);
- job.setReducerClass(SleepReducer.class);
- job.setOutputFormatClass(NullOutputFormat.class);
- job.setInputFormatClass(SleepInputFormat.class);
- job.setPartitionerClass(MRRSleepJobPartitioner.class);
- job.setSpeculativeExecution(false);
- job.setJobName("Sleep job");
-
- FileInputFormat.addInputPath(job, new Path("ignored"));
- return job;
- }
-
- public int run(String[] args) throws Exception {
-
- if(args.length < 1) {
- System.err.println("MRRSleepJob [-m numMapper] [-r numReducer]" +
- " [-ir numIntermediateReducer]" +
- " [-irs numIntermediateReducerStages]" +
- " [-mt mapSleepTime (msec)] [-rt reduceSleepTime (msec)]" +
- " [-irt intermediateReduceSleepTime]" +
- " [-recordt recordSleepTime (msec)]" +
- " [-generateSplitsInAM (false)/true]" +
- " [-writeSplitsToDfs (false)/true]");
- ToolRunner.printGenericCommandUsage(System.err);
- return 2;
- }
-
- int numMapper = 1, numReducer = 1, numIReducer = 1;
- long mapSleepTime = 100, reduceSleepTime = 100, recSleepTime = 100,
- iReduceSleepTime=1;
- int mapSleepCount = 1, reduceSleepCount = 1, iReduceSleepCount = 1;
- int iReduceStagesCount = 1;
- boolean writeSplitsToDfs = false;
- boolean generateSplitsInAM = false;
- boolean splitsOptionFound = false;
-
- for(int i=0; i < args.length; i++ ) {
- if(args[i].equals("-m")) {
- numMapper = Integer.parseInt(args[++i]);
- }
- else if(args[i].equals("-r")) {
- numReducer = Integer.parseInt(args[++i]);
- }
- else if(args[i].equals("-ir")) {
- numIReducer = Integer.parseInt(args[++i]);
- }
- else if(args[i].equals("-mt")) {
- mapSleepTime = Long.parseLong(args[++i]);
- }
- else if(args[i].equals("-rt")) {
- reduceSleepTime = Long.parseLong(args[++i]);
- }
- else if(args[i].equals("-irt")) {
- iReduceSleepTime = Long.parseLong(args[++i]);
- }
- else if(args[i].equals("-irs")) {
- iReduceStagesCount = Integer.parseInt(args[++i]);
- }
- else if (args[i].equals("-recordt")) {
- recSleepTime = Long.parseLong(args[++i]);
- }
- else if (args[i].equals("-generateSplitsInAM")) {
- if (splitsOptionFound) {
- throw new RuntimeException("Cannot use both -generateSplitsInAm and -writeSplitsToDfs together");
- }
- splitsOptionFound = true;
- generateSplitsInAM = Boolean.parseBoolean(args[++i]);
-
- }
- else if (args[i].equals("-writeSplitsToDfs")) {
- if (splitsOptionFound) {
- throw new RuntimeException("Cannot use both -generateSplitsInAm and -writeSplitsToDfs together");
- }
- splitsOptionFound = true;
- writeSplitsToDfs = Boolean.parseBoolean(args[++i]);
- }
- }
-
- if (numIReducer > 0 && numReducer <= 0) {
- throw new RuntimeException("Cannot have intermediate reduces without"
- + " a final reduce");
- }
-
- // sleep for *SleepTime duration in Task by recSleepTime per record
- mapSleepCount = (int)Math.ceil(mapSleepTime / ((double)recSleepTime));
- reduceSleepCount = (int)Math.ceil(reduceSleepTime / ((double)recSleepTime));
- iReduceSleepCount = (int)Math.ceil(iReduceSleepTime / ((double)recSleepTime));
-
- TezConfiguration conf = new TezConfiguration(getConf());
- FileSystem remoteFs = FileSystem.get(conf);
-
- conf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
- conf.get(
- TezConfiguration.TEZ_AM_STAGING_DIR,
- TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT));
-
- Path remoteStagingDir =
- remoteFs.makeQualified(new Path(conf.get(
- TezConfiguration.TEZ_AM_STAGING_DIR,
- TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT),
- Long.toString(System.currentTimeMillis())));
- TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);
-
- DAG dag = createDAG(remoteFs, conf, remoteStagingDir,
- numMapper, numReducer, iReduceStagesCount, numIReducer,
- mapSleepTime, mapSleepCount, reduceSleepTime, reduceSleepCount,
- iReduceSleepTime, iReduceSleepCount, writeSplitsToDfs, generateSplitsInAM);
-
- TezClient tezSession = new TezClient("MRRSleep", conf, false, null, credentials);
- tezSession.start();
- DAGClient dagClient = tezSession.submitDAG(dag);
-
- while (true) {
- DAGStatus status = dagClient.getDAGStatus(null);
- LOG.info("DAG Status: " + status);
- if (status.isCompleted()) {
- break;
- }
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- // do nothing
- }
- }
- tezSession.stop();
-
- return dagClient.getDAGStatus(null).getState().equals(DAGStatus.State.SUCCEEDED) ? 0 : 1;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MapredWordCount.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MapredWordCount.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MapredWordCount.java
deleted file mode 100644
index 6e7300c..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MapredWordCount.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.StringTokenizer;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MapReduceBase;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.tez.examples.WordCount;
-
-/**
- * This is an example Hadoop Map/Reduce application using the mapred apis.
- * It reads the text input files, breaks each line into words
- * and counts them. The output is a locally sorted list of words and the
- * count of how often they occurred.
- *
- * To run: bin/hadoop jar examples.jar wordcount
- * [-m <i>maps</i>] [-r <i>reduces</i>] <i>in-dir</i> <i>out-dir</i>
- */
-public class MapredWordCount extends Configured implements Tool {
-
- private static final Log LOG = LogFactory.getLog(MapredWordCount.class);
-
- /**
- * Counts the words in each line.
- * For each line of input, break the line into words and emit them as
- * (<b>word</b>, <b>1</b>).
- */
- public static class MapClass extends MapReduceBase
- implements Mapper<LongWritable, Text, Text, IntWritable> {
-
- private final static IntWritable one = new IntWritable(1);
- private Text word = new Text();
-
- public void map(LongWritable key, Text value,
- OutputCollector<Text, IntWritable> output,
- Reporter reporter) throws IOException {
- String line = value.toString();
- StringTokenizer itr = new StringTokenizer(line);
- while (itr.hasMoreTokens()) {
- word.set(itr.nextToken());
- output.collect(word, one);
- }
- }
- }
-
- /**
- * A reducer class that just emits the sum of the input values.
- */
- public static class Reduce extends MapReduceBase
- implements Reducer<Text, IntWritable, Text, IntWritable> {
-
- public void reduce(Text key, Iterator<IntWritable> values,
- OutputCollector<Text, IntWritable> output,
- Reporter reporter) throws IOException {
- int sum = 0;
- while (values.hasNext()) {
- sum += values.next().get();
- }
- output.collect(key, new IntWritable(sum));
- }
- }
-
- static int printUsage() {
- System.out.println("wordcount [-m <maps>] [-r <reduces>] <input> <output>");
- ToolRunner.printGenericCommandUsage(System.out);
- return -1;
- }
-
- /**
- * The main driver for word count map/reduce program.
- * Invoke this method to submit the map/reduce job.
- * @throws IOException When there is communication problems with the
- * job tracker.
- */
- public int run(String[] args) throws Exception {
- JobConf conf = new JobConf(getConf(), WordCount.class);
- conf.setJobName("wordcount");
- LOG.info("Running WordCount job using mapred apis");
-
- // the keys are words (strings)
- conf.setOutputKeyClass(Text.class);
- // the values are counts (ints)
- conf.setOutputValueClass(IntWritable.class);
-
- conf.setMapperClass(MapClass.class);
- conf.setCombinerClass(Reduce.class);
- conf.setReducerClass(Reduce.class);
-
- List<String> other_args = new ArrayList<String>();
- for(int i=0; i < args.length; ++i) {
- try {
- if ("-m".equals(args[i])) {
- conf.setNumMapTasks(Integer.parseInt(args[++i]));
- } else if ("-r".equals(args[i])) {
- conf.setNumReduceTasks(Integer.parseInt(args[++i]));
- } else {
- other_args.add(args[i]);
- }
- } catch (NumberFormatException except) {
- LOG.error("Integer expected instead of " + args[i]);
- return printUsage();
- } catch (ArrayIndexOutOfBoundsException except) {
- LOG.error("Required parameter missing from " + args[i-1]);
- return printUsage();
- }
- }
- // Make sure there are exactly 2 parameters left.
- if (other_args.size() != 2) {
- LOG.error("Wrong number of parameters: " +
- other_args.size() + " instead of 2.");
- return printUsage();
- }
- FileInputFormat.setInputPaths(conf, other_args.get(0));
- FileOutputFormat.setOutputPath(conf, new Path(other_args.get(1)));
-
- JobClient.runJob(conf);
- return 0;
- }
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(),
- new MapredWordCount(), args);
- System.exit(res);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tez/blob/41f5cd8a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java
deleted file mode 100644
index 8251b78..0000000
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java
+++ /dev/null
@@ -1,757 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.mapreduce.examples;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.ClusterStatus;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-/**
- * This program uses map/reduce to just run a distributed job where there is
- * no interaction between the tasks and each task writes a large unsorted
- * random sequence of words.
- * In order for this program to generate data for terasort with a 5-10 words
- * per key and 20-100 words per value, have the following config:
- * <xmp>
- * <?xml version="1.0"?>
- * <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
- * <configuration>
- * <property>
- * <name>mapreduce.randomtextwriter.minwordskey</name>
- * <value>5</value>
- * </property>
- * <property>
- * <name>mapreduce.randomtextwriter.maxwordskey</name>
- * <value>10</value>
- * </property>
- * <property>
- * <name>mapreduce.randomtextwriter.minwordsvalue</name>
- * <value>20</value>
- * </property>
- * <property>
- * <name>mapreduce.randomtextwriter.maxwordsvalue</name>
- * <value>100</value>
- * </property>
- * <property>
- * <name>mapreduce.randomtextwriter.totalbytes</name>
- * <value>1099511627776</value>
- * </property>
- * </configuration></xmp>
- *
- * Equivalently, {@link RandomTextWriter} also supports all the above options
- * and ones supported by {@link Tool} via the command-line.
- *
- * To run: bin/hadoop jar hadoop-${version}-examples.jar randomtextwriter
- * [-outFormat <i>output format class</i>] <i>output</i>
- */
-public class RandomTextWriter extends Configured implements Tool {
- public static final String TOTAL_BYTES =
- "mapreduce.randomtextwriter.totalbytes";
- public static final String BYTES_PER_MAP =
- "mapreduce.randomtextwriter.bytespermap";
- public static final String MAPS_PER_HOST =
- "mapreduce.randomtextwriter.mapsperhost";
- public static final String MAX_VALUE = "mapreduce.randomtextwriter.maxwordsvalue";
- public static final String MIN_VALUE = "mapreduce.randomtextwriter.minwordsvalue";
- public static final String MIN_KEY = "mapreduce.randomtextwriter.minwordskey";
- public static final String MAX_KEY = "mapreduce.randomtextwriter.maxwordskey";
-
- static int printUsage() {
- System.out.println("randomtextwriter " +
- "[-outFormat <output format class>] " +
- "<output>");
- ToolRunner.printGenericCommandUsage(System.out);
- return 2;
- }
-
- /**
- * User counters
- */
- static enum Counters { RECORDS_WRITTEN, BYTES_WRITTEN }
-
- static class RandomTextMapper extends Mapper<Text, Text, Text, Text> {
-
- private long numBytesToWrite;
- private int minWordsInKey;
- private int wordsInKeyRange;
- private int minWordsInValue;
- private int wordsInValueRange;
- private Random random = new Random();
-
- /**
- * Save the configuration value that we need to write the data.
- */
- public void setup(Context context) {
- Configuration conf = context.getConfiguration();
- numBytesToWrite = conf.getLong(BYTES_PER_MAP,
- 1*1024*1024*1024);
- minWordsInKey = conf.getInt(MIN_KEY, 5);
- wordsInKeyRange = (conf.getInt(MAX_KEY, 10) - minWordsInKey);
- minWordsInValue = conf.getInt(MIN_VALUE, 10);
- wordsInValueRange = (conf.getInt(MAX_VALUE, 100) - minWordsInValue);
- }
-
- /**
- * Given an output filename, write a bunch of random records to it.
- */
- public void map(Text key, Text value,
- Context context) throws IOException,InterruptedException {
- int itemCount = 0;
- while (numBytesToWrite > 0) {
- // Generate the key/value
- int noWordsKey = minWordsInKey +
- (wordsInKeyRange != 0 ? random.nextInt(wordsInKeyRange) : 0);
- int noWordsValue = minWordsInValue +
- (wordsInValueRange != 0 ? random.nextInt(wordsInValueRange) : 0);
- Text keyWords = generateSentence(noWordsKey);
- Text valueWords = generateSentence(noWordsValue);
-
- // Write the sentence
- context.write(keyWords, valueWords);
-
- numBytesToWrite -= (keyWords.getLength() + valueWords.getLength());
-
- // Update counters, progress etc.
- context.getCounter(Counters.BYTES_WRITTEN).increment(
- keyWords.getLength() + valueWords.getLength());
- context.getCounter(Counters.RECORDS_WRITTEN).increment(1);
- if (++itemCount % 200 == 0) {
- context.setStatus("wrote record " + itemCount + ". " +
- numBytesToWrite + " bytes left.");
- }
- }
- context.setStatus("done with " + itemCount + " records.");
- }
-
- private Text generateSentence(int noWords) {
- StringBuffer sentence = new StringBuffer();
- String space = " ";
- for (int i=0; i < noWords; ++i) {
- sentence.append(words[random.nextInt(words.length)]);
- sentence.append(space);
- }
- return new Text(sentence.toString());
- }
- }
-
- /**
- * This is the main routine for launching a distributed random write job.
- * It runs 10 maps/node and each node writes 1 gig of data to a DFS file.
- * The reduce doesn't do anything.
- *
- * @throws IOException
- */
- public int run(String[] args) throws Exception {
- if (args.length == 0) {
- return printUsage();
- }
-
- Configuration conf = getConf();
- JobClient client = new JobClient(conf);
- ClusterStatus cluster = client.getClusterStatus();
- int numMapsPerHost = conf.getInt(MAPS_PER_HOST, 10);
- long numBytesToWritePerMap = conf.getLong(BYTES_PER_MAP,
- 1*1024*1024*1024);
- if (numBytesToWritePerMap == 0) {
- System.err.println("Cannot have " + BYTES_PER_MAP +" set to 0");
- return -2;
- }
- long totalBytesToWrite = conf.getLong(TOTAL_BYTES,
- numMapsPerHost*numBytesToWritePerMap*cluster.getTaskTrackers());
- int numMaps = (int) (totalBytesToWrite / numBytesToWritePerMap);
- if (numMaps == 0 && totalBytesToWrite > 0) {
- numMaps = 1;
- conf.setLong(BYTES_PER_MAP, totalBytesToWrite);
- }
- conf.setInt(MRJobConfig.NUM_MAPS, numMaps);
-
- Job job = new Job(conf);
-
- job.setJarByClass(RandomTextWriter.class);
- job.setJobName("random-text-writer");
-
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(Text.class);
-
- job.setInputFormatClass(RandomWriter.RandomInputFormat.class);
- job.setMapperClass(RandomTextMapper.class);
-
- Class<? extends OutputFormat> outputFormatClass =
- SequenceFileOutputFormat.class;
- List<String> otherArgs = new ArrayList<String>();
- for(int i=0; i < args.length; ++i) {
- try {
- if ("-outFormat".equals(args[i])) {
- outputFormatClass =
- Class.forName(args[++i]).asSubclass(OutputFormat.class);
- } else {
- otherArgs.add(args[i]);
- }
- } catch (ArrayIndexOutOfBoundsException except) {
- System.out.println("ERROR: Required parameter missing from " +
- args[i-1]);
- return printUsage(); // exits
- }
- }
-
- job.setOutputFormatClass(outputFormatClass);
- FileOutputFormat.setOutputPath(job, new Path(otherArgs.get(0)));
-
- System.out.println("Running " + numMaps + " maps.");
-
- // reducer NONE
- job.setNumReduceTasks(0);
-
- Date startTime = new Date();
- System.out.println("Job started: " + startTime);
- int ret = job.waitForCompletion(true) ? 0 : 1;
- Date endTime = new Date();
- System.out.println("Job ended: " + endTime);
- System.out.println("The job took " +
- (endTime.getTime() - startTime.getTime()) /1000 +
- " seconds.");
-
- return ret;
- }
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new RandomTextWriter(), args);
- System.exit(res);
- }
-
- /**
- * A random list of 100 words from /usr/share/dict/words
- */
- private static String[] words = {
- "diurnalness", "Homoiousian",
- "spiranthic", "tetragynian",
- "silverhead", "ungreat",
- "lithograph", "exploiter",
- "physiologian", "by",
- "hellbender", "Filipendula",
- "undeterring", "antiscolic",
- "pentagamist", "hypoid",
- "cacuminal", "sertularian",
- "schoolmasterism", "nonuple",
- "gallybeggar", "phytonic",
- "swearingly", "nebular",
- "Confervales", "thermochemically",
- "characinoid", "cocksuredom",
- "fallacious", "feasibleness",
- "debromination", "playfellowship",
- "tramplike", "testa",
- "participatingly", "unaccessible",
- "bromate", "experientialist",
- "roughcast", "docimastical",
- "choralcelo", "blightbird",
- "peptonate", "sombreroed",
- "unschematized", "antiabolitionist",
- "besagne", "mastication",
- "bromic", "sviatonosite",
- "cattimandoo", "metaphrastical",
- "endotheliomyoma", "hysterolysis",
- "unfulminated", "Hester",
- "oblongly", "blurredness",
- "authorling", "chasmy",
- "Scorpaenidae", "toxihaemia",
- "Dictograph", "Quakerishly",
- "deaf", "timbermonger",
- "strammel", "Thraupidae",
- "seditious", "plerome",
- "Arneb", "eristically",
- "serpentinic", "glaumrie",
- "socioromantic", "apocalypst",
- "tartrous", "Bassaris",
- "angiolymphoma", "horsefly",
- "kenno", "astronomize",
- "euphemious", "arsenide",
- "untongued", "parabolicness",
- "uvanite", "helpless",
- "gemmeous", "stormy",
- "templar", "erythrodextrin",
- "comism", "interfraternal",
- "preparative", "parastas",
- "frontoorbital", "Ophiosaurus",
- "diopside", "serosanguineous",
- "ununiformly", "karyological",
- "collegian", "allotropic",
- "depravity", "amylogenesis",
- "reformatory", "epidymides",
- "pleurotropous", "trillium",
- "dastardliness", "coadvice",
- "embryotic", "benthonic",
- "pomiferous", "figureheadship",
- "Megaluridae", "Harpa",
- "frenal", "commotion",
- "abthainry", "cobeliever",
- "manilla", "spiciferous",
- "nativeness", "obispo",
- "monilioid", "biopsic",
- "valvula", "enterostomy",
- "planosubulate", "pterostigma",
- "lifter", "triradiated",
- "venialness", "tum",
- "archistome", "tautness",
- "unswanlike", "antivenin",
- "Lentibulariaceae", "Triphora",
- "angiopathy", "anta",
- "Dawsonia", "becomma",
- "Yannigan", "winterproof",
- "antalgol", "harr",
- "underogating", "ineunt",
- "cornberry", "flippantness",
- "scyphostoma", "approbation",
- "Ghent", "Macraucheniidae",
- "scabbiness", "unanatomized",
- "photoelasticity", "eurythermal",
- "enation", "prepavement",
- "flushgate", "subsequentially",
- "Edo", "antihero",
- "Isokontae", "unforkedness",
- "porriginous", "daytime",
- "nonexecutive", "trisilicic",
- "morphiomania", "paranephros",
- "botchedly", "impugnation",
- "Dodecatheon", "obolus",
- "unburnt", "provedore",
- "Aktistetae", "superindifference",
- "Alethea", "Joachimite",
- "cyanophilous", "chorograph",
- "brooky", "figured",
- "periclitation", "quintette",
- "hondo", "ornithodelphous",
- "unefficient", "pondside",
- "bogydom", "laurinoxylon",
- "Shiah", "unharmed",
- "cartful", "noncrystallized",
- "abusiveness", "cromlech",
- "japanned", "rizzomed",
- "underskin", "adscendent",
- "allectory", "gelatinousness",
- "volcano", "uncompromisingly",
- "cubit", "idiotize",
- "unfurbelowed", "undinted",
- "magnetooptics", "Savitar",
- "diwata", "ramosopalmate",
- "Pishquow", "tomorn",
- "apopenptic", "Haversian",
- "Hysterocarpus", "ten",
- "outhue", "Bertat",
- "mechanist", "asparaginic",
- "velaric", "tonsure",
- "bubble", "Pyrales",
- "regardful", "glyphography",
- "calabazilla", "shellworker",
- "stradametrical", "havoc",
- "theologicopolitical", "sawdust",
- "diatomaceous", "jajman",
- "temporomastoid", "Serrifera",
- "Ochnaceae", "aspersor",
- "trailmaking", "Bishareen",
- "digitule", "octogynous",
- "epididymitis", "smokefarthings",
- "bacillite", "overcrown",
- "mangonism", "sirrah",
- "undecorated", "psychofugal",
- "bismuthiferous", "rechar",
- "Lemuridae", "frameable",
- "thiodiazole", "Scanic",
- "sportswomanship", "interruptedness",
- "admissory", "osteopaedion",
- "tingly", "tomorrowness",
- "ethnocracy", "trabecular",
- "vitally", "fossilism",
- "adz", "metopon",
- "prefatorial", "expiscate",
- "diathermacy", "chronist",
- "nigh", "generalizable",
- "hysterogen", "aurothiosulphuric",
- "whitlowwort", "downthrust",
- "Protestantize", "monander",
- "Itea", "chronographic",
- "silicize", "Dunlop",
- "eer", "componental",
- "spot", "pamphlet",
- "antineuritic", "paradisean",
- "interruptor", "debellator",
- "overcultured", "Florissant",
- "hyocholic", "pneumatotherapy",
- "tailoress", "rave",
- "unpeople", "Sebastian",
- "thermanesthesia", "Coniferae",
- "swacking", "posterishness",
- "ethmopalatal", "whittle",
- "analgize", "scabbardless",
- "naught", "symbiogenetically",
- "trip", "parodist",
- "columniform", "trunnel",
- "yawler", "goodwill",
- "pseudohalogen", "swangy",
- "cervisial", "mediateness",
- "genii", "imprescribable",
- "pony", "consumptional",
- "carposporangial", "poleax",
- "bestill", "subfebrile",
- "sapphiric", "arrowworm",
- "qualminess", "ultraobscure",
- "thorite", "Fouquieria",
- "Bermudian", "prescriber",
- "elemicin", "warlike",
- "semiangle", "rotular",
- "misthread", "returnability",
- "seraphism", "precostal",
- "quarried", "Babylonism",
- "sangaree", "seelful",
- "placatory", "pachydermous",
- "bozal", "galbulus",
- "spermaphyte", "cumbrousness",
- "pope", "signifier",
- "Endomycetaceae", "shallowish",
- "sequacity", "periarthritis",
- "bathysphere", "pentosuria",
- "Dadaism", "spookdom",
- "Consolamentum", "afterpressure",
- "mutter", "louse",
- "ovoviviparous", "corbel",
- "metastoma", "biventer",
- "Hydrangea", "hogmace",
- "seizing", "nonsuppressed",
- "oratorize", "uncarefully",
- "benzothiofuran", "penult",
- "balanocele", "macropterous",
- "dishpan", "marten",
- "absvolt", "jirble",
- "parmelioid", "airfreighter",
- "acocotl", "archesporial",
- "hypoplastral", "preoral",
- "quailberry", "cinque",
- "terrestrially", "stroking",
- "limpet", "moodishness",
- "canicule", "archididascalian",
- "pompiloid", "overstaid",
- "introducer", "Italical",
- "Christianopaganism", "prescriptible",
- "subofficer", "danseuse",
- "cloy", "saguran",
- "frictionlessly", "deindividualization",
- "Bulanda", "ventricous",
- "subfoliar", "basto",
- "scapuloradial", "suspend",
- "stiffish", "Sphenodontidae",
- "eternal", "verbid",
- "mammonish", "upcushion",
- "barkometer", "concretion",
- "preagitate", "incomprehensible",
- "tristich", "visceral",
- "hemimelus", "patroller",
- "stentorophonic", "pinulus",
- "kerykeion", "brutism",
- "monstership", "merciful",
- "overinstruct", "defensibly",
- "bettermost", "splenauxe",
- "Mormyrus", "unreprimanded",
- "taver", "ell",
- "proacquittal", "infestation",
- "overwoven", "Lincolnlike",
- "chacona", "Tamil",
- "classificational", "lebensraum",
- "reeveland", "intuition",
- "Whilkut", "focaloid",
- "Eleusinian", "micromembrane",
- "byroad", "nonrepetition",
- "bacterioblast", "brag",
- "ribaldrous", "phytoma",
- "counteralliance", "pelvimetry",
- "pelf", "relaster",
- "thermoresistant", "aneurism",
- "molossic", "euphonym",
- "upswell", "ladhood",
- "phallaceous", "inertly",
- "gunshop", "stereotypography",
- "laryngic", "refasten",
- "twinling", "oflete",
- "hepatorrhaphy", "electrotechnics",
- "cockal", "guitarist",
- "topsail", "Cimmerianism",
- "larklike", "Llandovery",
- "pyrocatechol", "immatchable",
- "chooser", "metrocratic",
- "craglike", "quadrennial",
- "nonpoisonous", "undercolored",
- "knob", "ultratense",
- "balladmonger", "slait",
- "sialadenitis", "bucketer",
- "magnificently", "unstipulated",
- "unscourged", "unsupercilious",
- "packsack", "pansophism",
- "soorkee", "percent",
- "subirrigate", "champer",
- "metapolitics", "spherulitic",
- "involatile", "metaphonical",
- "stachyuraceous", "speckedness",
- "bespin", "proboscidiform",
- "gul", "squit",
- "yeelaman", "peristeropode",
- "opacousness", "shibuichi",
- "retinize", "yote",
- "misexposition", "devilwise",
- "pumpkinification", "vinny",
- "bonze", "glossing",
- "decardinalize", "transcortical",
- "serphoid", "deepmost",
- "guanajuatite", "wemless",
- "arval", "lammy",
- "Effie", "Saponaria",
- "tetrahedral", "prolificy",
- "excerpt", "dunkadoo",
- "Spencerism", "insatiately",
- "Gilaki", "oratorship",
- "arduousness", "unbashfulness",
- "Pithecolobium", "unisexuality",
- "veterinarian", "detractive",
- "liquidity", "acidophile",
- "proauction", "sural",
- "totaquina", "Vichyite",
- "uninhabitedness", "allegedly",
- "Gothish", "manny",
- "Inger", "flutist",
- "ticktick", "Ludgatian",
- "homotransplant", "orthopedical",
- "diminutively", "monogoneutic",
- "Kenipsim", "sarcologist",
- "drome", "stronghearted",
- "Fameuse", "Swaziland",
- "alen", "chilblain",
- "beatable", "agglomeratic",
- "constitutor", "tendomucoid",
- "porencephalous", "arteriasis",
- "boser", "tantivy",
- "rede", "lineamental",
- "uncontradictableness", "homeotypical",
- "masa", "folious",
- "dosseret", "neurodegenerative",
- "subtransverse", "Chiasmodontidae",
- "palaeotheriodont", "unstressedly",
- "chalcites", "piquantness",
- "lampyrine", "Aplacentalia",
- "projecting", "elastivity",
- "isopelletierin", "bladderwort",
- "strander", "almud",
- "iniquitously", "theologal",
- "bugre", "chargeably",
- "imperceptivity", "meriquinoidal",
- "mesophyte", "divinator",
- "perfunctory", "counterappellant",
- "synovial", "charioteer",
- "crystallographical", "comprovincial",
- "infrastapedial", "pleasurehood",
- "inventurous", "ultrasystematic",
- "subangulated", "supraoesophageal",
- "Vaishnavism", "transude",
- "chrysochrous", "ungrave",
- "reconciliable", "uninterpleaded",
- "erlking", "wherefrom",
- "aprosopia", "antiadiaphorist",
- "metoxazine", "incalculable",
- "umbellic", "predebit",
- "foursquare", "unimmortal",
- "nonmanufacture", "slangy",
- "predisputant", "familist",
- "preaffiliate", "friarhood",
- "corelysis", "zoonitic",
- "halloo", "paunchy",
- "neuromimesis", "aconitine",
- "hackneyed", "unfeeble",
- "cubby", "autoschediastical",
- "naprapath", "lyrebird",
- "inexistency", "leucophoenicite",
- "ferrogoslarite", "reperuse",
- "uncombable", "tambo",
- "propodiale", "diplomatize",
- "Russifier", "clanned",
- "corona", "michigan",
- "nonutilitarian", "transcorporeal",
- "bought", "Cercosporella",
- "stapedius", "glandularly",
- "pictorially", "weism",
- "disilane", "rainproof",
- "Caphtor", "scrubbed",
- "oinomancy", "pseudoxanthine",
- "nonlustrous", "redesertion",
- "Oryzorictinae", "gala",
- "Mycogone", "reappreciate",
- "cyanoguanidine", "seeingness",
- "breadwinner", "noreast",
- "furacious", "epauliere",
- "omniscribent", "Passiflorales",
- "uninductive", "inductivity",
- "Orbitolina", "Semecarpus",
- "migrainoid", "steprelationship",
- "phlogisticate", "mesymnion",
- "sloped", "edificator",
- "beneficent", "culm",
- "paleornithology", "unurban",
- "throbless", "amplexifoliate",
- "sesquiquintile", "sapience",
- "astucious", "dithery",
- "boor", "ambitus",
- "scotching", "uloid",
- "uncompromisingness", "hoove",
- "waird", "marshiness",
- "Jerusalem", "mericarp",
- "unevoked", "benzoperoxide",
- "outguess", "pyxie",
- "hymnic", "euphemize",
- "mendacity", "erythremia",
- "rosaniline", "unchatteled",
- "lienteria", "Bushongo",
- "dialoguer", "unrepealably",
- "rivethead", "antideflation",
- "vinegarish", "manganosiderite",
- "doubtingness", "ovopyriform",
- "Cephalodiscus", "Muscicapa",
- "Animalivora", "angina",
- "planispheric", "ipomoein",
- "cuproiodargyrite", "sandbox",
- "scrat", "Munnopsidae",
- "shola", "pentafid",
- "overstudiousness", "times",
- "nonprofession", "appetible",
- "valvulotomy", "goladar",
- "uniarticular", "oxyterpene",
- "unlapsing", "omega",
- "trophonema", "seminonflammable",
- "circumzenithal", "starer",
- "depthwise", "liberatress",
- "unleavened", "unrevolting",
- "groundneedle", "topline",
- "wandoo", "umangite",
- "ordinant", "unachievable",
- "oversand", "snare",
- "avengeful", "unexplicit",
- "mustafina", "sonable",
- "rehabilitative", "eulogization",
- "papery", "technopsychology",
- "impressor", "cresylite",
- "entame", "transudatory",
- "scotale", "pachydermatoid",
- "imaginary", "yeat",
- "slipped", "stewardship",
- "adatom", "cockstone",
- "skyshine", "heavenful",
- "comparability", "exprobratory",
- "dermorhynchous", "parquet",
- "cretaceous", "vesperal",
- "raphis", "undangered",
- "Glecoma", "engrain",
- "counteractively", "Zuludom",
- "orchiocatabasis", "Auriculariales",
- "warriorwise", "extraorganismal",
- "overbuilt", "alveolite",
- "tetchy", "terrificness",
- "widdle", "unpremonished",
- "rebilling", "sequestrum",
- "equiconvex", "heliocentricism",
- "catabaptist", "okonite",
- "propheticism", "helminthagogic",
- "calycular", "giantly",
- "wingable", "golem",
- "unprovided", "commandingness",
- "greave", "haply",
- "doina", "depressingly",
- "subdentate", "impairment",
- "decidable", "neurotrophic",
- "unpredict", "bicorporeal",
- "pendulant", "flatman",
- "intrabred", "toplike",
- "Prosobranchiata", "farrantly",
- "toxoplasmosis", "gorilloid",
- "dipsomaniacal", "aquiline",
- "atlantite", "ascitic",
- "perculsive", "prospectiveness",
- "saponaceous", "centrifugalization",
- "dinical", "infravaginal",
- "beadroll", "affaite",
- "Helvidian", "tickleproof",
- "abstractionism", "enhedge",
- "outwealth", "overcontribute",
- "coldfinch", "gymnastic",
- "Pincian", "Munychian",
- "codisjunct", "quad",
- "coracomandibular", "phoenicochroite",
- "amender", "selectivity",
- "putative", "semantician",
- "lophotrichic", "Spatangoidea",
- "saccharogenic", "inferent",
- "Triconodonta", "arrendation",
- "sheepskin", "taurocolla",
- "bunghole", "Machiavel",
- "triakistetrahedral", "dehairer",
- "prezygapophysial", "cylindric",
- "pneumonalgia", "sleigher",
- "emir", "Socraticism",
- "licitness", "massedly",
- "instructiveness", "sturdied",
- "redecrease", "starosta",
- "evictor", "orgiastic",
- "squdge", "meloplasty",
- "Tsonecan", "repealableness",
- "swoony", "myesthesia",
- "molecule", "autobiographist",
- "reciprocation", "refective",
- "unobservantness", "tricae",
- "ungouged", "floatability",
- "Mesua", "fetlocked",
- "chordacentrum", "sedentariness",
- "various", "laubanite",
- "nectopod", "zenick",
- "sequentially", "analgic",
- "biodynamics", "posttraumatic",
- "nummi", "pyroacetic",
- "bot", "redescend",
- "dispermy", "undiffusive",
- "circular", "trillion",
- "Uraniidae", "ploration",
- "discipular", "potentness",
- "sud", "Hu",
- "Eryon", "plugger",
- "subdrainage", "jharal",
- "abscission", "supermarket",
- "countergabion", "glacierist",
- "lithotresis", "minniebush",
- "zanyism", "eucalypteol",
- "sterilely", "unrealize",
- "unpatched", "hypochondriacism",
- "critically", "cheesecutter",
- };
-}