You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by bi...@apache.org on 2014/08/16 05:05:23 UTC
[2/6] TEZ-1132. Consistent naming of Input and Outputs (bikas)
http://git-wip-us.apache.org/repos/asf/tez/blob/469bf905/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedPartitionedKVOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedPartitionedKVOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedPartitionedKVOutput.java
deleted file mode 100644
index e11c2e4..0000000
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileUnorderedPartitionedKVOutput.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.runtime.library.output;
-
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import com.google.common.base.Preconditions;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.common.TezRuntimeFrameworkConfigs;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.runtime.api.AbstractLogicalOutput;
-import org.apache.tez.runtime.api.Event;
-import org.apache.tez.runtime.api.LogicalOutput;
-import org.apache.tez.runtime.api.OutputContext;
-import org.apache.tez.runtime.api.Writer;
-import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
-import org.apache.tez.runtime.library.common.MemoryUpdateCallbackHandler;
-import org.apache.tez.runtime.library.common.writers.UnorderedPartitionedKVWriter;
-
-/**
- * <code>OnFileUnorderedPartitionedKVOutput</code> is a {@link LogicalOutput} which can be used to
- * write Key-Value pairs. The key-value pairs are written to the correct partition based on the
- * configured Partitioner.
- */
-public class OnFileUnorderedPartitionedKVOutput extends AbstractLogicalOutput {
-
- private static final Log LOG = LogFactory.getLog(OnFileUnorderedPartitionedKVOutput.class);
-
- private Configuration conf;
- private MemoryUpdateCallbackHandler memoryUpdateCallbackHandler;
- private UnorderedPartitionedKVWriter kvWriter;
- private final AtomicBoolean isStarted = new AtomicBoolean(false);
-
- public OnFileUnorderedPartitionedKVOutput(OutputContext outputContext, int numPhysicalOutputs) {
- super(outputContext, numPhysicalOutputs);
- }
-
- @Override
- public synchronized List<Event> initialize() throws Exception {
- this.conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());
- this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs());
- this.conf.setInt(TezRuntimeFrameworkConfigs.TEZ_RUNTIME_NUM_EXPECTED_PARTITIONS,
- getNumPhysicalOutputs());
- this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
- getContext().requestInitialMemory(
- UnorderedPartitionedKVWriter.getInitialMemoryRequirement(conf,
- getContext().getTotalMemoryAvailableToTask()), memoryUpdateCallbackHandler);
- return Collections.emptyList();
- }
-
- @Override
- public synchronized void start() throws Exception {
- if (!isStarted.get()) {
- memoryUpdateCallbackHandler.validateUpdateReceived();
- this.kvWriter = new UnorderedPartitionedKVWriter(getContext(), conf, getNumPhysicalOutputs(),
- memoryUpdateCallbackHandler.getMemoryAssigned());
- isStarted.set(true);
- }
- }
-
- @Override
- public synchronized Writer getWriter() throws Exception {
- Preconditions.checkState(isStarted.get(), "Cannot get writer before starting the Output");
- return kvWriter;
- }
-
- @Override
- public void handleEvents(List<Event> outputEvents) {
- }
-
- @Override
- public synchronized List<Event> close() throws Exception {
- if (isStarted.get()) {
- return kvWriter.close();
- } else {
- return Collections.emptyList();
- }
- }
-
- private static final Set<String> confKeys = new HashSet<String>();
-
- static {
- confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD);
- confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES);
- confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE);
- confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES);
- confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB);
- confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES);
- confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS);
- confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS);
- confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS);
- confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS);
- confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC);
- confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED);
- confKeys.add(TezConfiguration.TEZ_AM_COUNTERS_MAX_KEYS);
- confKeys.add(TezConfiguration.TEZ_AM_COUNTERS_GROUP_NAME_MAX_KEYS);
- confKeys.add(TezConfiguration.TEZ_AM_COUNTERS_NAME_MAX_KEYS);
- confKeys.add(TezConfiguration.TEZ_AM_COUNTERS_GROUPS_MAX_KEYS);
- }
-
- // TODO Maybe add helper methods to extract keys
- // TODO Maybe add constants or an Enum to access the keys
-
- @InterfaceAudience.Private
- public static Set<String> getConfigurationKeySet() {
- return Collections.unmodifiableSet(confKeys);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/469bf905/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java
new file mode 100644
index 0000000..3c32ec8
--- /dev/null
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java
@@ -0,0 +1,249 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.runtime.library.output;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.tez.common.TezCommonUtils;
+import org.apache.tez.common.TezRuntimeFrameworkConfigs;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.common.TezUtilsInternal;
+import org.apache.tez.common.counters.TaskCounter;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.runtime.api.AbstractLogicalOutput;
+import org.apache.tez.runtime.api.Event;
+import org.apache.tez.runtime.api.OutputContext;
+import org.apache.tez.runtime.api.events.CompositeDataMovementEvent;
+import org.apache.tez.runtime.api.events.VertexManagerEvent;
+import org.apache.tez.runtime.library.api.KeyValuesWriter;
+import org.apache.tez.runtime.library.api.Partitioner;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
+import org.apache.tez.runtime.library.common.MemoryUpdateCallbackHandler;
+import org.apache.tez.runtime.library.common.sort.impl.ExternalSorter;
+import org.apache.tez.runtime.library.common.sort.impl.PipelinedSorter;
+import org.apache.tez.runtime.library.common.sort.impl.TezIndexRecord;
+import org.apache.tez.runtime.library.common.sort.impl.TezSpillRecord;
+import org.apache.tez.runtime.library.common.sort.impl.dflt.DefaultSorter;
+import org.apache.tez.runtime.library.shuffle.common.ShuffleUtils;
+import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.DataMovementEventPayloadProto;
+import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.VertexManagerEventPayloadProto;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.protobuf.ByteString;
+
+/**
+ * {@link OrderedPartitionedKVOutput} is an {@link AbstractLogicalOutput} which sorts
+ * key/value pairs written to it. It also partitions the output based on a
+ * {@link Partitioner}
+ */
+public class OrderedPartitionedKVOutput extends AbstractLogicalOutput {
+
+ private static final Log LOG = LogFactory.getLog(OrderedPartitionedKVOutput.class);
+
+ protected ExternalSorter sorter;
+ protected Configuration conf;
+ protected MemoryUpdateCallbackHandler memoryUpdateCallbackHandler;
+ private long startTime;
+ private long endTime;
+ private boolean sendEmptyPartitionDetails;
+ private final AtomicBoolean isStarted = new AtomicBoolean(false);
+
+ public OrderedPartitionedKVOutput(OutputContext outputContext, int numPhysicalOutputs) {
+ super(outputContext, numPhysicalOutputs);
+ }
+
+ @Override
+ public synchronized List<Event> initialize() throws IOException {
+ this.startTime = System.nanoTime();
+ this.conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());
+ // Initializing this parametr in this conf since it is used in multiple
+ // places (wherever LocalDirAllocator is used) - TezTaskOutputFiles,
+ // TezMerger, etc.
+ this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs());
+ this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
+ getContext().requestInitialMemory(
+ ExternalSorter.getInitialMemoryRequirement(conf,
+ getContext().getTotalMemoryAvailableToTask()), memoryUpdateCallbackHandler);
+
+ sendEmptyPartitionDetails = this.conf.getBoolean(
+ TezRuntimeConfiguration.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED,
+ TezRuntimeConfiguration.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED_DEFAULT);
+ return Collections.emptyList();
+ }
+
+ @Override
+ public synchronized void start() throws Exception {
+ if (!isStarted.get()) {
+ memoryUpdateCallbackHandler.validateUpdateReceived();
+ if (this.conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_THREADS,
+ TezRuntimeConfiguration.TEZ_RUNTIME_SORT_THREADS_DEFAULT) > 1) {
+ sorter = new PipelinedSorter(getContext(), conf, getNumPhysicalOutputs(),
+ memoryUpdateCallbackHandler.getMemoryAssigned());
+ } else {
+ sorter = new DefaultSorter(getContext(), conf, getNumPhysicalOutputs(),
+ memoryUpdateCallbackHandler.getMemoryAssigned());
+ }
+ isStarted.set(true);
+ }
+ }
+
+ @Override
+ public synchronized KeyValuesWriter getWriter() throws IOException {
+ Preconditions.checkState(isStarted.get(), "Cannot get writer before starting the Output");
+ return new KeyValuesWriter() {
+ @Override
+ public void write(Object key, Object value) throws IOException {
+ sorter.write(key, value);
+ }
+
+ @Override
+ public void write(Object key, Iterable<Object> values) throws IOException {
+ sorter.write(key, values);
+ }
+ };
+ }
+
+ @Override
+ public synchronized void handleEvents(List<Event> outputEvents) {
+ // Not expecting any events.
+ }
+
+ @Override
+ public synchronized List<Event> close() throws IOException {
+ if (sorter != null) {
+ sorter.flush();
+ sorter.close();
+ this.endTime = System.nanoTime();
+ return generateEventsOnClose();
+ } else {
+ LOG.warn("Attempting to close output " + getContext().getDestinationVertexName()
+ + " before it was started");
+ return Collections.emptyList();
+ }
+ }
+
+ protected List<Event> generateEventsOnClose() throws IOException {
+ DataMovementEventPayloadProto.Builder payloadBuilder = DataMovementEventPayloadProto
+ .newBuilder();
+
+ boolean outputGenerated = true;
+ if (sendEmptyPartitionDetails) {
+ Path indexFile = sorter.getMapOutput().getOutputIndexFile();
+ TezSpillRecord spillRecord = new TezSpillRecord(indexFile, conf);
+ BitSet emptyPartitionDetails = new BitSet();
+ int emptyPartitions = 0;
+ for(int i=0;i<spillRecord.size();i++) {
+ TezIndexRecord indexRecord = spillRecord.getIndex(i);
+ if (!indexRecord.hasData()) {
+ emptyPartitionDetails.set(i);
+ emptyPartitions++;
+ }
+ }
+ outputGenerated = (spillRecord.size() != emptyPartitions);
+ if (emptyPartitions > 0) {
+ ByteString emptyPartitionsBytesString =
+ TezCommonUtils.compressByteArrayToByteString(
+ TezUtilsInternal.toByteArray(emptyPartitionDetails));
+ payloadBuilder.setEmptyPartitions(emptyPartitionsBytesString);
+ LOG.info("EmptyPartition bitsetSize=" + emptyPartitionDetails.cardinality() + ", numOutputs="
+ + getNumPhysicalOutputs() + ", emptyPartitions=" + emptyPartitions
+ + ", compressedSize=" + emptyPartitionsBytesString.size());
+ }
+ }
+ if (!sendEmptyPartitionDetails || outputGenerated) {
+ String host = System.getenv(ApplicationConstants.Environment.NM_HOST
+ .toString());
+ ByteBuffer shuffleMetadata = getContext()
+ .getServiceProviderMetaData(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID);
+ int shufflePort = ShuffleUtils.deserializeShuffleProviderMetaData(shuffleMetadata);
+ payloadBuilder.setHost(host);
+ payloadBuilder.setPort(shufflePort);
+ payloadBuilder.setPathComponent(getContext().getUniqueIdentifier());
+ }
+
+ payloadBuilder.setRunDuration((int) ((endTime - startTime) / 1000));
+ DataMovementEventPayloadProto payloadProto = payloadBuilder.build();
+ byte[] payload = payloadProto.toByteArray();
+
+ long outputSize = getContext().getCounters()
+ .findCounter(TaskCounter.OUTPUT_BYTES).getValue();
+ VertexManagerEventPayloadProto.Builder vmBuilder = VertexManagerEventPayloadProto
+ .newBuilder();
+ vmBuilder.setOutputSize(outputSize);
+ VertexManagerEvent vmEvent = new VertexManagerEvent(
+ getContext().getDestinationVertexName(), vmBuilder.build().toByteArray());
+
+ List<Event> events = Lists.newArrayListWithCapacity(getNumPhysicalOutputs() + 1);
+ events.add(vmEvent);
+
+ CompositeDataMovementEvent csdme = new CompositeDataMovementEvent(0, getNumPhysicalOutputs(), payload);
+ events.add(csdme);
+
+ return events;
+ }
+
+
+ private static final Set<String> confKeys = new HashSet<String>();
+
+ static {
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINE_MIN_SPILLS);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_THREADS);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_COMPARATOR_CLASS);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED);
+ confKeys.add(TezConfiguration.TEZ_AM_COUNTERS_MAX_KEYS);
+ confKeys.add(TezConfiguration.TEZ_AM_COUNTERS_GROUP_NAME_MAX_KEYS);
+ confKeys.add(TezConfiguration.TEZ_AM_COUNTERS_NAME_MAX_KEYS);
+ confKeys.add(TezConfiguration.TEZ_AM_COUNTERS_GROUPS_MAX_KEYS);
+ }
+
+ // TODO Maybe add helper methods to extract keys
+ // TODO Maybe add constants or an Enum to access the keys
+
+ @InterfaceAudience.Private
+ public static Set<String> getConfigurationKeySet() {
+ return Collections.unmodifiableSet(confKeys);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/469bf905/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java
new file mode 100644
index 0000000..71c9d52
--- /dev/null
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java
@@ -0,0 +1,193 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.runtime.library.output;
+
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.common.TezCommonUtils;
+import org.apache.tez.common.TezRuntimeFrameworkConfigs;
+import org.apache.tez.common.TezUtilsInternal;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.runtime.api.AbstractLogicalOutput;
+import org.apache.tez.runtime.api.Event;
+import org.apache.tez.runtime.api.LogicalOutput;
+import org.apache.tez.runtime.api.OutputContext;
+import org.apache.tez.runtime.api.events.DataMovementEvent;
+import org.apache.tez.runtime.library.api.KeyValuesWriter;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
+import org.apache.tez.runtime.library.broadcast.output.FileBasedKVWriter;
+import org.apache.tez.runtime.library.shuffle.common.ShuffleUtils;
+import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.DataMovementEventPayloadProto;
+import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.DataProto;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+import com.google.protobuf.ByteString;
+
+/**
+ * {@link UnorderedKVOutput} is a {@link LogicalOutput} that writes key
+ * value data without applying any ordering or grouping constraints. This can be
+ * used to write raw key value data as is.
+ */
+public class UnorderedKVOutput extends AbstractLogicalOutput {
+
+ private static final Log LOG = LogFactory.getLog(UnorderedKVOutput.class);
+
+ private FileBasedKVWriter kvWriter;
+
+ private Configuration conf;
+
+ private boolean dataViaEventsEnabled;
+ private int dataViaEventsMaxSize;
+
+ public UnorderedKVOutput(OutputContext outputContext, int numPhysicalOutputs) {
+ super(outputContext, numPhysicalOutputs);
+ }
+
+
+ @Override
+ public synchronized List<Event> initialize()
+ throws Exception {
+ this.conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());
+ this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS,
+ getContext().getWorkDirs());
+
+ getContext().requestInitialMemory(0l, null); // mandatory call
+
+ this.dataViaEventsEnabled = conf.getBoolean(
+ TezRuntimeConfiguration.TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_ENABLED,
+ TezRuntimeConfiguration.TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_ENABLED_DEFAULT);
+ this.dataViaEventsMaxSize = conf.getInt(
+ TezRuntimeConfiguration.TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_MAX_SIZE,
+ TezRuntimeConfiguration.TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_MAX_SIZE_DEFAULT);
+
+ LOG.info(this.getClass().getSimpleName() + " running with params -> "
+ + "dataViaEventsEnabled: " + dataViaEventsEnabled
+ + ", dataViaEventsMaxSize: " + dataViaEventsMaxSize);
+
+ this.kvWriter = new FileBasedKVWriter(getContext(), conf);
+ return Collections.emptyList();
+ }
+
+ @Override
+ public synchronized void start() {
+ }
+
+ @Override
+ public synchronized KeyValuesWriter getWriter() throws Exception {
+ // Eventually, disallow multiple invocations.
+ return kvWriter;
+ }
+
+ @Override
+ public synchronized void handleEvents(List<Event> outputEvents) {
+ throw new TezUncheckedException("Not expecting any events");
+ }
+
+ @Override
+ public synchronized List<Event> close() throws Exception {
+ boolean outputGenerated = this.kvWriter.close();
+
+ DataMovementEventPayloadProto.Builder payloadBuilder = DataMovementEventPayloadProto
+ .newBuilder();
+
+ LOG.info("Closing KVOutput: RawLength: " + this.kvWriter.getRawLength()
+ + ", CompressedLength: " + this.kvWriter.getCompressedLength());
+
+ if (dataViaEventsEnabled && outputGenerated && this.kvWriter.getCompressedLength() <= dataViaEventsMaxSize) {
+ LOG.info("Serialzing actual data into DataMovementEvent, dataSize: " + this.kvWriter.getCompressedLength());
+ byte[] data = this.kvWriter.getData();
+ DataProto.Builder dataProtoBuilder = DataProto.newBuilder();
+ dataProtoBuilder.setData(ByteString.copyFrom(data));
+ dataProtoBuilder.setRawLength((int)this.kvWriter.getRawLength());
+ dataProtoBuilder.setCompressedLength((int)this.kvWriter.getCompressedLength());
+ payloadBuilder.setData(dataProtoBuilder.build());
+ }
+
+ // Set the list of empty partitions - single partition on this case.
+ if (!outputGenerated) {
+ LOG.info("No output was generated");
+ BitSet emptyPartitions = new BitSet();
+ emptyPartitions.set(0);
+ ByteString emptyPartitionsBytesString =
+ TezCommonUtils.compressByteArrayToByteString(TezUtilsInternal.toByteArray(emptyPartitions));
+ payloadBuilder.setEmptyPartitions(emptyPartitionsBytesString);
+ }
+ if (outputGenerated) {
+ String host = getHost();
+ ByteBuffer shuffleMetadata = getContext()
+ .getServiceProviderMetaData(ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID);
+ int shufflePort = ShuffleUtils
+ .deserializeShuffleProviderMetaData(shuffleMetadata);
+ payloadBuilder.setHost(host);
+ payloadBuilder.setPort(shufflePort);
+ payloadBuilder.setPathComponent(getContext().getUniqueIdentifier());
+ }
+ DataMovementEventPayloadProto payloadProto = payloadBuilder.build();
+
+ DataMovementEvent dmEvent = new DataMovementEvent(0, payloadProto.toByteArray());
+ List<Event> events = Lists.newArrayListWithCapacity(1);
+ events.add(dmEvent);
+ return events;
+ }
+
+ @VisibleForTesting
+ @Private
+ String getHost() {
+ return System.getenv(ApplicationConstants.Environment.NM_HOST.toString());
+ }
+
+ private static final Set<String> confKeys = new HashSet<String>();
+
+ static {
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED);
+ confKeys.add(TezConfiguration.TEZ_AM_COUNTERS_MAX_KEYS);
+ confKeys.add(TezConfiguration.TEZ_AM_COUNTERS_GROUP_NAME_MAX_KEYS);
+ confKeys.add(TezConfiguration.TEZ_AM_COUNTERS_NAME_MAX_KEYS);
+ confKeys.add(TezConfiguration.TEZ_AM_COUNTERS_GROUPS_MAX_KEYS);
+ }
+
+ // TODO Maybe add helper methods to extract keys
+ // TODO Maybe add constants or an Enum to access the keys
+
+ @InterfaceAudience.Private
+ public static Set<String> getConfigurationKeySet() {
+ return Collections.unmodifiableSet(confKeys);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/469bf905/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedPartitionedKVOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedPartitionedKVOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedPartitionedKVOutput.java
new file mode 100644
index 0000000..8d4ed3a
--- /dev/null
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedPartitionedKVOutput.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.runtime.library.output;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.common.TezRuntimeFrameworkConfigs;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.runtime.api.AbstractLogicalOutput;
+import org.apache.tez.runtime.api.Event;
+import org.apache.tez.runtime.api.LogicalOutput;
+import org.apache.tez.runtime.api.OutputContext;
+import org.apache.tez.runtime.api.Writer;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
+import org.apache.tez.runtime.library.common.MemoryUpdateCallbackHandler;
+import org.apache.tez.runtime.library.common.writers.UnorderedPartitionedKVWriter;
+
+/**
+ * {@link UnorderedPartitionedKVOutput} is a {@link LogicalOutput} which can be used to
+ * write Key-Value pairs. The key-value pairs are written to the correct partition based on the
+ * configured Partitioner.
+ */
+public class UnorderedPartitionedKVOutput extends AbstractLogicalOutput {
+
+ private static final Log LOG = LogFactory.getLog(UnorderedPartitionedKVOutput.class);
+
+ private Configuration conf;
+ private MemoryUpdateCallbackHandler memoryUpdateCallbackHandler;
+ private UnorderedPartitionedKVWriter kvWriter;
+ private final AtomicBoolean isStarted = new AtomicBoolean(false);
+
+ public UnorderedPartitionedKVOutput(OutputContext outputContext, int numPhysicalOutputs) {
+ super(outputContext, numPhysicalOutputs);
+ }
+
+ @Override
+ public synchronized List<Event> initialize() throws Exception {
+ this.conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());
+ this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs());
+ this.conf.setInt(TezRuntimeFrameworkConfigs.TEZ_RUNTIME_NUM_EXPECTED_PARTITIONS,
+ getNumPhysicalOutputs());
+ this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
+ getContext().requestInitialMemory(
+ UnorderedPartitionedKVWriter.getInitialMemoryRequirement(conf,
+ getContext().getTotalMemoryAvailableToTask()), memoryUpdateCallbackHandler);
+ return Collections.emptyList();
+ }
+
+ @Override
+ public synchronized void start() throws Exception {
+ if (!isStarted.get()) {
+ memoryUpdateCallbackHandler.validateUpdateReceived();
+ this.kvWriter = new UnorderedPartitionedKVWriter(getContext(), conf, getNumPhysicalOutputs(),
+ memoryUpdateCallbackHandler.getMemoryAssigned());
+ isStarted.set(true);
+ }
+ }
+
+ @Override
+ public synchronized Writer getWriter() throws Exception {
+ Preconditions.checkState(isStarted.get(), "Cannot get writer before starting the Output");
+ return kvWriter;
+ }
+
+ @Override
+ public void handleEvents(List<Event> outputEvents) {
+ }
+
+ @Override
+ public synchronized List<Event> close() throws Exception {
+ if (isStarted.get()) {
+ return kvWriter.close();
+ } else {
+ return Collections.emptyList();
+ }
+ }
+
+ private static final Set<String> confKeys = new HashSet<String>();
+
+ static {
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED);
+ confKeys.add(TezConfiguration.TEZ_AM_COUNTERS_MAX_KEYS);
+ confKeys.add(TezConfiguration.TEZ_AM_COUNTERS_GROUP_NAME_MAX_KEYS);
+ confKeys.add(TezConfiguration.TEZ_AM_COUNTERS_NAME_MAX_KEYS);
+ confKeys.add(TezConfiguration.TEZ_AM_COUNTERS_GROUPS_MAX_KEYS);
+ }
+
+ // TODO Maybe add helper methods to extract keys
+ // TODO Maybe add constants or an Enum to access the keys
+
+ @InterfaceAudience.Private
+ public static Set<String> getConfigurationKeySet() {
+ return Collections.unmodifiableSet(confKeys);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/469bf905/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/resources/WeightedScalingMemoryDistributor.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/resources/WeightedScalingMemoryDistributor.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/resources/WeightedScalingMemoryDistributor.java
index 55010d1..c6eeced 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/resources/WeightedScalingMemoryDistributor.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/resources/WeightedScalingMemoryDistributor.java
@@ -33,11 +33,11 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.runtime.common.resources.InitialMemoryAllocator;
import org.apache.tez.runtime.common.resources.InitialMemoryRequestContext;
-import org.apache.tez.runtime.library.input.ShuffledMergedInput;
-import org.apache.tez.runtime.library.input.ShuffledMergedInputLegacy;
-import org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput;
-import org.apache.tez.runtime.library.output.OnFileSortedOutput;
-import org.apache.tez.runtime.library.output.OnFileUnorderedPartitionedKVOutput;
+import org.apache.tez.runtime.library.input.OrderedGroupedKVInput;
+import org.apache.tez.runtime.library.input.OrderedGroupedInputLegacy;
+import org.apache.tez.runtime.library.input.UnorderedKVInput;
+import org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput;
+import org.apache.tez.runtime.library.output.UnorderedPartitionedKVOutput;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -180,14 +180,14 @@ public class WeightedScalingMemoryDistributor implements InitialMemoryAllocator
private RequestType getRequestTypeForClass(String className) {
RequestType requestType;
- if (className.equals(OnFileSortedOutput.class.getName())) {
+ if (className.equals(OrderedPartitionedKVOutput.class.getName())) {
requestType = RequestType.SORTED_OUTPUT;
- } else if (className.equals(ShuffledMergedInput.class.getName())
- || className.equals(ShuffledMergedInputLegacy.class.getName())) {
+ } else if (className.equals(OrderedGroupedKVInput.class.getName())
+ || className.equals(OrderedGroupedInputLegacy.class.getName())) {
requestType = RequestType.SORTED_MERGED_INPUT;
- } else if (className.equals(ShuffledUnorderedKVInput.class.getName())) {
+ } else if (className.equals(UnorderedKVInput.class.getName())) {
requestType = RequestType.UNSORTED_INPUT;
- } else if (className.equals(OnFileUnorderedPartitionedKVOutput.class.getName())) {
+ } else if (className.equals(UnorderedPartitionedKVOutput.class.getName())) {
requestType = RequestType.PARTITIONED_UNSORTED_OUTPUT;
} else {
requestType = RequestType.OTHER;
http://git-wip-us.apache.org/repos/asf/tez/blob/469bf905/tez-runtime-library/src/test/java/org/apache/tez/runtime/common/resources/TestWeightedScalingMemoryDistributor.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/common/resources/TestWeightedScalingMemoryDistributor.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/common/resources/TestWeightedScalingMemoryDistributor.java
index 137b1f4..e6888fd 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/common/resources/TestWeightedScalingMemoryDistributor.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/common/resources/TestWeightedScalingMemoryDistributor.java
@@ -31,9 +31,9 @@ import org.apache.tez.runtime.api.LogicalOutput;
import org.apache.tez.runtime.api.MemoryUpdateCallback;
import org.apache.tez.runtime.api.InputContext;
import org.apache.tez.runtime.api.OutputContext;
-import org.apache.tez.runtime.library.input.ShuffledMergedInput;
-import org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput;
-import org.apache.tez.runtime.library.output.OnFileSortedOutput;
+import org.apache.tez.runtime.library.input.OrderedGroupedKVInput;
+import org.apache.tez.runtime.library.input.UnorderedKVInput;
+import org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput;
import org.apache.tez.runtime.library.resources.WeightedScalingMemoryDistributor;
import org.apache.tez.runtime.library.resources.WeightedScalingMemoryDistributor.RequestType;
import org.junit.Test;
@@ -66,13 +66,13 @@ public class TestWeightedScalingMemoryDistributor extends TestMemoryDistributor
// First request - ScatterGatherShuffleInput
MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
InputContext e1InputContext1 = createTestInputContext();
- InputDescriptor e1InDesc1 = createTestInputDescriptor(ShuffledMergedInput.class);
+ InputDescriptor e1InDesc1 = createTestInputDescriptor(OrderedGroupedKVInput.class);
dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);
// Second request - BroadcastInput
MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
InputContext e2InputContext2 = createTestInputContext();
- InputDescriptor e2InDesc2 = createTestInputDescriptor(ShuffledUnorderedKVInput.class);
+ InputDescriptor e2InDesc2 = createTestInputDescriptor(UnorderedKVInput.class);
dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
// Third request - randomOutput (simulates MROutput)
@@ -84,7 +84,7 @@ public class TestWeightedScalingMemoryDistributor extends TestMemoryDistributor
// Fourth request - OnFileSortedOutput
MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
OutputContext e4OutputContext2 = createTestOutputContext();
- OutputDescriptor e4OutDesc2 = createTestOutputDescriptor(OnFileSortedOutput.class);
+ OutputDescriptor e4OutDesc2 = createTestOutputDescriptor(OrderedPartitionedKVOutput.class);
dist.requestMemory(10000, e4Callback, e4OutputContext2, e4OutDesc2);
dist.makeInitialAllocations();
@@ -113,13 +113,13 @@ public class TestWeightedScalingMemoryDistributor extends TestMemoryDistributor
// First request - ScatterGatherShuffleInput [weight 6]
MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
InputContext e1InputContext1 = createTestInputContext();
- InputDescriptor e1InDesc1 = createTestInputDescriptor(ShuffledMergedInput.class);
+ InputDescriptor e1InDesc1 = createTestInputDescriptor(OrderedGroupedKVInput.class);
dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);
// Second request - BroadcastInput [weight 2]
MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
InputContext e2InputContext2 = createTestInputContext();
- InputDescriptor e2InDesc2 = createTestInputDescriptor(ShuffledUnorderedKVInput.class);
+ InputDescriptor e2InDesc2 = createTestInputDescriptor(UnorderedKVInput.class);
dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
// Third request - randomOutput (simulates MROutput) [weight 1]
@@ -131,7 +131,7 @@ public class TestWeightedScalingMemoryDistributor extends TestMemoryDistributor
// Fourth request - OnFileSortedOutput [weight 3]
MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
OutputContext e4OutputContext2 = createTestOutputContext();
- OutputDescriptor e4OutDesc2 = createTestOutputDescriptor(OnFileSortedOutput.class);
+ OutputDescriptor e4OutDesc2 = createTestOutputDescriptor(OrderedPartitionedKVOutput.class);
dist.requestMemory(10000, e4Callback, e4OutputContext2, e4OutDesc2);
dist.makeInitialAllocations();
http://git-wip-us.apache.org/repos/asf/tez/blob/469bf905/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOnFileSortedOutputConfigurer.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOnFileSortedOutputConfigurer.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOnFileSortedOutputConfigurer.java
index bc20195..970f500 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOnFileSortedOutputConfigurer.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOnFileSortedOutputConfigurer.java
@@ -38,7 +38,7 @@ public class TestOnFileSortedOutputConfigurer {
@Test
public void testNullParams() {
try {
- OnFileSortedOutputConfigurer.newBuilder(
+ OrderedPartitionedKVOutputConfigurer.newBuilder(
null, "VALUE", "PARTITIONER", null);
fail("Expecting a null parameter list to fail");
} catch (NullPointerException npe) {
@@ -46,7 +46,7 @@ public class TestOnFileSortedOutputConfigurer {
}
try {
- OnFileSortedOutputConfigurer.newBuilder(
+ OrderedPartitionedKVOutputConfigurer.newBuilder(
"KEY", null, "PARTITIONER", null);
fail("Expecting a null parameter list to fail");
} catch (NullPointerException npe) {
@@ -54,7 +54,7 @@ public class TestOnFileSortedOutputConfigurer {
}
try {
- OnFileSortedOutputConfigurer.newBuilder(
+ OrderedPartitionedKVOutputConfigurer.newBuilder(
"KEY", "VALUE", null, null);
fail("Expecting a null parameter list to fail");
} catch (NullPointerException npe) {
@@ -72,8 +72,8 @@ public class TestOnFileSortedOutputConfigurer {
additionalConf.put("test.key.2", "key2");
additionalConf.put("io.shouldExist", "io");
additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "TestInternalSorter");
- OnFileSortedOutputConfigurer.Builder builder =
- OnFileSortedOutputConfigurer.newBuilder("KEY", "VALUE", "PARTITIONER", null)
+ OrderedPartitionedKVOutputConfigurer.Builder builder =
+ OrderedPartitionedKVOutputConfigurer.newBuilder("KEY", "VALUE", "PARTITIONER", null)
.setKeyComparatorClass("KEY_COMPARATOR", null)
.setCompression(true, "CustomCodec")
.setSortBufferSize(2048)
@@ -84,9 +84,9 @@ public class TestOnFileSortedOutputConfigurer {
.setAdditionalConfiguration(additionalConf)
.setFromConfiguration(fromConf);
- OnFileSortedOutputConfigurer configuration = builder.build();
+ OrderedPartitionedKVOutputConfigurer configuration = builder.build();
- OnFileSortedOutputConfigurer rebuilt = new OnFileSortedOutputConfigurer();
+ OrderedPartitionedKVOutputConfigurer rebuilt = new OrderedPartitionedKVOutputConfigurer();
rebuilt.fromUserPayload(configuration.toUserPayload());
Configuration conf = rebuilt.conf;
@@ -119,11 +119,11 @@ public class TestOnFileSortedOutputConfigurer {
@Test
public void testDefaultConfigsUsed() {
- OnFileSortedOutputConfigurer.Builder builder =
- OnFileSortedOutputConfigurer.newBuilder("KEY", "VALUE", "PARTITIONER", null);
- OnFileSortedOutputConfigurer configuration = builder.build();
+ OrderedPartitionedKVOutputConfigurer.Builder builder =
+ OrderedPartitionedKVOutputConfigurer.newBuilder("KEY", "VALUE", "PARTITIONER", null);
+ OrderedPartitionedKVOutputConfigurer configuration = builder.build();
- OnFileSortedOutputConfigurer rebuilt = new OnFileSortedOutputConfigurer();
+ OrderedPartitionedKVOutputConfigurer rebuilt = new OrderedPartitionedKVOutputConfigurer();
rebuilt.fromUserPayload(configuration.toUserPayload());
Configuration conf = rebuilt.conf;
@@ -147,12 +147,12 @@ public class TestOnFileSortedOutputConfigurer {
partitionerConf.put("partitioner.test.key", "PARTITIONERKEY");
partitionerConf
.put(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "InvalidKeyOverride");
- OnFileSortedOutputConfigurer.Builder builder =
- OnFileSortedOutputConfigurer.newBuilder("KEY", "VALUE", "PARTITIONER", partitionerConf);
+ OrderedPartitionedKVOutputConfigurer.Builder builder =
+ OrderedPartitionedKVOutputConfigurer.newBuilder("KEY", "VALUE", "PARTITIONER", partitionerConf);
- OnFileSortedOutputConfigurer configuration = builder.build();
+ OrderedPartitionedKVOutputConfigurer configuration = builder.build();
- OnFileSortedOutputConfigurer rebuilt = new OnFileSortedOutputConfigurer();
+ OrderedPartitionedKVOutputConfigurer rebuilt = new OrderedPartitionedKVOutputConfigurer();
rebuilt.fromUserPayload(configuration.toUserPayload());
Configuration conf = rebuilt.conf;
@@ -170,13 +170,13 @@ public class TestOnFileSortedOutputConfigurer {
combinerConf.put("combiner.test.key", "COMBINERKEY");
combinerConf
.put(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "InvalidKeyOverride");
- OnFileSortedOutputConfigurer.Builder builder =
- OnFileSortedOutputConfigurer.newBuilder("KEY", "VALUE", "PARTITIONER", null)
+ OrderedPartitionedKVOutputConfigurer.Builder builder =
+ OrderedPartitionedKVOutputConfigurer.newBuilder("KEY", "VALUE", "PARTITIONER", null)
.setCombiner("COMBINER", combinerConf);
- OnFileSortedOutputConfigurer configuration = builder.build();
+ OrderedPartitionedKVOutputConfigurer configuration = builder.build();
- OnFileSortedOutputConfigurer rebuilt = new OnFileSortedOutputConfigurer();
+ OrderedPartitionedKVOutputConfigurer rebuilt = new OrderedPartitionedKVOutputConfigurer();
rebuilt.fromUserPayload(configuration.toUserPayload());
Configuration conf = rebuilt.conf;
http://git-wip-us.apache.org/repos/asf/tez/blob/469bf905/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOnFileUnorderedKVOutputConfigurer.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOnFileUnorderedKVOutputConfigurer.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOnFileUnorderedKVOutputConfigurer.java
index b26aafb..2422203 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOnFileUnorderedKVOutputConfigurer.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOnFileUnorderedKVOutputConfigurer.java
@@ -37,7 +37,7 @@ public class TestOnFileUnorderedKVOutputConfigurer {
@Test
public void testNullParams() {
try {
- OnFileUnorderedKVOutputConfigurer.newBuilder(
+ UnorderedKVOutputConfigurer.newBuilder(
null, "VALUE");
fail("Expecting a null parameter list to fail");
} catch (NullPointerException npe) {
@@ -45,7 +45,7 @@ public class TestOnFileUnorderedKVOutputConfigurer {
}
try {
- OnFileUnorderedKVOutputConfigurer.newBuilder(
+ UnorderedKVOutputConfigurer.newBuilder(
"KEY", null);
fail("Expecting a null parameter list to fail");
} catch (NullPointerException npe) {
@@ -63,8 +63,8 @@ public class TestOnFileUnorderedKVOutputConfigurer {
additionalConf.put("test.key.2", "key2");
additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES, "2222");
additionalConf.put("file.shouldExist", "file");
- OnFileUnorderedKVOutputConfigurer.Builder builder =
- OnFileUnorderedKVOutputConfigurer.newBuilder("KEY", "VALUE")
+ UnorderedKVOutputConfigurer.Builder builder =
+ UnorderedKVOutputConfigurer.newBuilder("KEY", "VALUE")
.setCompression(true, "CustomCodec")
.setAdditionalConfiguration("fs.shouldExist", "fs")
.setAdditionalConfiguration("test.key.1", "key1")
@@ -73,10 +73,10 @@ public class TestOnFileUnorderedKVOutputConfigurer {
.setAdditionalConfiguration(additionalConf)
.setFromConfiguration(fromConf);
- OnFileUnorderedKVOutputConfigurer configuration = builder.build();
+ UnorderedKVOutputConfigurer configuration = builder.build();
- OnFileUnorderedKVOutputConfigurer rebuilt =
- new OnFileUnorderedKVOutputConfigurer();
+ UnorderedKVOutputConfigurer rebuilt =
+ new UnorderedKVOutputConfigurer();
rebuilt.fromUserPayload(configuration.toUserPayload());
Configuration conf = rebuilt.conf;
@@ -104,13 +104,13 @@ public class TestOnFileUnorderedKVOutputConfigurer {
@Test
public void testDefaultConfigsUsed() {
- OnFileUnorderedKVOutputConfigurer.Builder builder =
- OnFileUnorderedKVOutputConfigurer
+ UnorderedKVOutputConfigurer.Builder builder =
+ UnorderedKVOutputConfigurer
.newBuilder("KEY", "VALUE");
- OnFileUnorderedKVOutputConfigurer configuration = builder.build();
+ UnorderedKVOutputConfigurer configuration = builder.build();
- OnFileUnorderedKVOutputConfigurer rebuilt =
- new OnFileUnorderedKVOutputConfigurer();
+ UnorderedKVOutputConfigurer rebuilt =
+ new UnorderedKVOutputConfigurer();
rebuilt.fromUserPayload(configuration.toUserPayload());
Configuration conf = rebuilt.conf;
http://git-wip-us.apache.org/repos/asf/tez/blob/469bf905/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOnFileUnorderedPartitionedKVOutputConfigurer.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOnFileUnorderedPartitionedKVOutputConfigurer.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOnFileUnorderedPartitionedKVOutputConfigurer.java
index d12c5eb..73cab11 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOnFileUnorderedPartitionedKVOutputConfigurer.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOnFileUnorderedPartitionedKVOutputConfigurer.java
@@ -39,7 +39,7 @@ public class TestOnFileUnorderedPartitionedKVOutputConfigurer {
@Test
public void testNullParams() {
try {
- OnFileUnorderedPartitionedKVOutputConfigurer.newBuilder(
+ UnorderedPartitionedKVOutputConfigurer.newBuilder(
null, "VALUE", "PARTITIONER", null);
fail("Expecting a null parameter list to fail");
} catch (NullPointerException npe) {
@@ -47,7 +47,7 @@ public class TestOnFileUnorderedPartitionedKVOutputConfigurer {
}
try {
- OnFileUnorderedPartitionedKVOutputConfigurer.newBuilder(
+ UnorderedPartitionedKVOutputConfigurer.newBuilder(
"KEY", null, "PARTITIONER", null);
fail("Expecting a null parameter list to fail");
} catch (NullPointerException npe) {
@@ -55,7 +55,7 @@ public class TestOnFileUnorderedPartitionedKVOutputConfigurer {
}
try {
- OnFileUnorderedPartitionedKVOutputConfigurer.newBuilder(
+ UnorderedPartitionedKVOutputConfigurer.newBuilder(
"KEY", "VALUE", null, null);
fail("Expecting a null parameter list to fail");
} catch (NullPointerException npe) {
@@ -73,8 +73,8 @@ public class TestOnFileUnorderedPartitionedKVOutputConfigurer {
additionalConf.put("test.key.2", "key2");
additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES, "2222");
additionalConf.put("file.shouldExist", "file");
- OnFileUnorderedPartitionedKVOutputConfigurer.Builder builder =
- OnFileUnorderedPartitionedKVOutputConfigurer.newBuilder("KEY", "VALUE", "PARTITIONER",
+ UnorderedPartitionedKVOutputConfigurer.Builder builder =
+ UnorderedPartitionedKVOutputConfigurer.newBuilder("KEY", "VALUE", "PARTITIONER",
null)
.setCompression(true, "CustomCodec")
.setAvailableBufferSize(1111)
@@ -85,10 +85,10 @@ public class TestOnFileUnorderedPartitionedKVOutputConfigurer {
.setAdditionalConfiguration(additionalConf)
.setFromConfiguration(fromConf);
- OnFileUnorderedPartitionedKVOutputConfigurer configuration = builder.build();
+ UnorderedPartitionedKVOutputConfigurer configuration = builder.build();
- OnFileUnorderedPartitionedKVOutputConfigurer rebuilt =
- new OnFileUnorderedPartitionedKVOutputConfigurer();
+ UnorderedPartitionedKVOutputConfigurer rebuilt =
+ new UnorderedPartitionedKVOutputConfigurer();
rebuilt.fromUserPayload(configuration.toUserPayload());
Configuration conf = rebuilt.conf;
@@ -120,15 +120,15 @@ public class TestOnFileUnorderedPartitionedKVOutputConfigurer {
@Test
public void testDefaultConfigsUsed() {
- OnFileUnorderedPartitionedKVOutputConfigurer.Builder builder =
- OnFileUnorderedPartitionedKVOutputConfigurer
+ UnorderedPartitionedKVOutputConfigurer.Builder builder =
+ UnorderedPartitionedKVOutputConfigurer
.newBuilder("KEY", "VALUE", "PARTITIONER", null)
.setKeySerializationClass("SerClass1")
.setValueSerializationClass("SerClass2");
- OnFileUnorderedPartitionedKVOutputConfigurer configuration = builder.build();
+ UnorderedPartitionedKVOutputConfigurer configuration = builder.build();
- OnFileUnorderedPartitionedKVOutputConfigurer rebuilt =
- new OnFileUnorderedPartitionedKVOutputConfigurer();
+ UnorderedPartitionedKVOutputConfigurer rebuilt =
+ new UnorderedPartitionedKVOutputConfigurer();
rebuilt.fromUserPayload(configuration.toUserPayload());
Configuration conf = rebuilt.conf;
@@ -156,14 +156,14 @@ public class TestOnFileUnorderedPartitionedKVOutputConfigurer {
partitionerConf.put("partitioner.test.key", "PARTITIONERKEY");
partitionerConf
.put(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "InvalidKeyOverride");
- OnFileUnorderedPartitionedKVOutputConfigurer.Builder builder =
- OnFileUnorderedPartitionedKVOutputConfigurer
+ UnorderedPartitionedKVOutputConfigurer.Builder builder =
+ UnorderedPartitionedKVOutputConfigurer
.newBuilder("KEY", "VALUE", "PARTITIONER", partitionerConf);
- OnFileUnorderedPartitionedKVOutputConfigurer configuration = builder.build();
+ UnorderedPartitionedKVOutputConfigurer configuration = builder.build();
- OnFileUnorderedPartitionedKVOutputConfigurer rebuilt =
- new OnFileUnorderedPartitionedKVOutputConfigurer();
+ UnorderedPartitionedKVOutputConfigurer rebuilt =
+ new UnorderedPartitionedKVOutputConfigurer();
rebuilt.fromUserPayload(configuration.toUserPayload());
Configuration conf = rebuilt.conf;
http://git-wip-us.apache.org/repos/asf/tez/blob/469bf905/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfigurer.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfigurer.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfigurer.java
index b0f9c5c..6f91cbc 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfigurer.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfigurer.java
@@ -67,9 +67,9 @@ public class TestOrderedPartitionedKVEdgeConfigurer {
OrderedPartitionedKVEdgeConfigurer configuration = builder.build();
- OnFileSortedOutputConfigurer rebuiltOutput = new OnFileSortedOutputConfigurer();
+ OrderedPartitionedKVOutputConfigurer rebuiltOutput = new OrderedPartitionedKVOutputConfigurer();
rebuiltOutput.fromUserPayload(configuration.getOutputPayload());
- ShuffledMergedInputConfigurer rebuiltInput = new ShuffledMergedInputConfigurer();
+ OrderedGroupedKVInputConfigurer rebuiltInput = new OrderedGroupedKVInputConfigurer();
rebuiltInput.fromUserPayload(configuration.getInputPayload());
Configuration outputConf = rebuiltOutput.conf;
@@ -93,9 +93,9 @@ public class TestOrderedPartitionedKVEdgeConfigurer {
OrderedPartitionedKVEdgeConfigurer configuration = builder.build();
- OnFileSortedOutputConfigurer rebuiltOutput = new OnFileSortedOutputConfigurer();
+ OrderedPartitionedKVOutputConfigurer rebuiltOutput = new OrderedPartitionedKVOutputConfigurer();
rebuiltOutput.fromUserPayload(configuration.getOutputPayload());
- ShuffledMergedInputConfigurer rebuiltInput = new ShuffledMergedInputConfigurer();
+ OrderedGroupedKVInputConfigurer rebuiltInput = new OrderedGroupedKVInputConfigurer();
rebuiltInput.fromUserPayload(configuration.getInputPayload());
Configuration outputConf = rebuiltOutput.conf;
@@ -135,9 +135,9 @@ public class TestOrderedPartitionedKVEdgeConfigurer {
OrderedPartitionedKVEdgeConfigurer configuration = builder.build();
- OnFileSortedOutputConfigurer rebuiltOutput = new OnFileSortedOutputConfigurer();
+ OrderedPartitionedKVOutputConfigurer rebuiltOutput = new OrderedPartitionedKVOutputConfigurer();
rebuiltOutput.fromUserPayload(configuration.getOutputPayload());
- ShuffledMergedInputConfigurer rebuiltInput = new ShuffledMergedInputConfigurer();
+ OrderedGroupedKVInputConfigurer rebuiltInput = new OrderedGroupedKVInputConfigurer();
rebuiltInput.fromUserPayload(configuration.getInputPayload());
Configuration outputConf = rebuiltOutput.conf;
@@ -190,9 +190,9 @@ public class TestOrderedPartitionedKVEdgeConfigurer {
OrderedPartitionedKVEdgeConfigurer configuration = builder.build();
- OnFileSortedOutputConfigurer rebuiltOutput = new OnFileSortedOutputConfigurer();
+ OrderedPartitionedKVOutputConfigurer rebuiltOutput = new OrderedPartitionedKVOutputConfigurer();
rebuiltOutput.fromUserPayload(configuration.getOutputPayload());
- ShuffledMergedInputConfigurer rebuiltInput = new ShuffledMergedInputConfigurer();
+ OrderedGroupedKVInputConfigurer rebuiltInput = new OrderedGroupedKVInputConfigurer();
rebuiltInput.fromUserPayload(configuration.getInputPayload());
Configuration outputConf = rebuiltOutput.conf;
@@ -250,9 +250,9 @@ public class TestOrderedPartitionedKVEdgeConfigurer {
OrderedPartitionedKVEdgeConfigurer configuration = builder.build();
- OnFileSortedOutputConfigurer rebuiltOutput = new OnFileSortedOutputConfigurer();
+ OrderedPartitionedKVOutputConfigurer rebuiltOutput = new OrderedPartitionedKVOutputConfigurer();
rebuiltOutput.fromUserPayload(configuration.getOutputPayload());
- ShuffledMergedInputConfigurer rebuiltInput = new ShuffledMergedInputConfigurer();
+ OrderedGroupedKVInputConfigurer rebuiltInput = new OrderedGroupedKVInputConfigurer();
rebuiltInput.fromUserPayload(configuration.getInputPayload());
Configuration outputConf = rebuiltOutput.conf;
http://git-wip-us.apache.org/repos/asf/tez/blob/469bf905/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestShuffledMergedInputConfigurer.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestShuffledMergedInputConfigurer.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestShuffledMergedInputConfigurer.java
index af91773..4560556 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestShuffledMergedInputConfigurer.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestShuffledMergedInputConfigurer.java
@@ -38,14 +38,14 @@ public class TestShuffledMergedInputConfigurer {
@Test
public void testNullParams() {
try {
- ShuffledMergedInputConfigurer.newBuilder(null, "VALUE");
+ OrderedGroupedKVInputConfigurer.newBuilder(null, "VALUE");
fail("Expecting a null parameter list to fail");
} catch (NullPointerException npe) {
assertTrue(npe.getMessage().contains("cannot be null"));
}
try {
- ShuffledMergedInputConfigurer.newBuilder("KEY", null);
+ OrderedGroupedKVInputConfigurer.newBuilder("KEY", null);
fail("Expecting a null parameter list to fail");
} catch (NullPointerException npe) {
assertTrue(npe.getMessage().contains("cannot be null"));
@@ -62,8 +62,8 @@ public class TestShuffledMergedInputConfigurer {
additionalConf.put("test.key.2", "key2");
additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, "3");
additionalConf.put("file.shouldExist", "file");
- ShuffledMergedInputConfigurer.Builder builder =
- ShuffledMergedInputConfigurer.newBuilder("KEY", "VALUE")
+ OrderedGroupedKVInputConfigurer.Builder builder =
+ OrderedGroupedKVInputConfigurer.newBuilder("KEY", "VALUE")
.setKeyComparatorClass("KEY_COMPARATOR", null)
.setCompression(true, "CustomCodec")
.setMaxSingleMemorySegmentFraction(0.11f)
@@ -77,10 +77,10 @@ public class TestShuffledMergedInputConfigurer {
.setAdditionalConfiguration(additionalConf)
.setFromConfiguration(fromConf);
- ShuffledMergedInputConfigurer configuration = builder.build();
+ OrderedGroupedKVInputConfigurer configuration = builder.build();
- ShuffledMergedInputConfigurer rebuilt = new ShuffledMergedInputConfigurer();
+ OrderedGroupedKVInputConfigurer rebuilt = new OrderedGroupedKVInputConfigurer();
rebuilt.fromUserPayload(configuration.toUserPayload());
Configuration conf = rebuilt.conf;
@@ -113,11 +113,11 @@ public class TestShuffledMergedInputConfigurer {
@Test
public void testDefaultConfigsUsed() {
- ShuffledMergedInputConfigurer.Builder builder =
- ShuffledMergedInputConfigurer.newBuilder("KEY", "VALUE");
- ShuffledMergedInputConfigurer configuration = builder.build();
+ OrderedGroupedKVInputConfigurer.Builder builder =
+ OrderedGroupedKVInputConfigurer.newBuilder("KEY", "VALUE");
+ OrderedGroupedKVInputConfigurer configuration = builder.build();
- ShuffledMergedInputConfigurer rebuilt = new ShuffledMergedInputConfigurer();
+ OrderedGroupedKVInputConfigurer rebuilt = new OrderedGroupedKVInputConfigurer();
rebuilt.fromUserPayload(configuration.toUserPayload());
Configuration conf = rebuilt.conf;
@@ -140,13 +140,13 @@ public class TestShuffledMergedInputConfigurer {
combinerConf.put("combiner.test.key", "COMBINERKEY");
combinerConf
.put(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "InvalidKeyOverride");
- ShuffledMergedInputConfigurer.Builder builder =
- ShuffledMergedInputConfigurer.newBuilder("KEY", "VALUE")
+ OrderedGroupedKVInputConfigurer.Builder builder =
+ OrderedGroupedKVInputConfigurer.newBuilder("KEY", "VALUE")
.setCombiner("COMBINER", combinerConf);
- ShuffledMergedInputConfigurer configuration = builder.build();
+ OrderedGroupedKVInputConfigurer configuration = builder.build();
- ShuffledMergedInputConfigurer rebuilt = new ShuffledMergedInputConfigurer();
+ OrderedGroupedKVInputConfigurer rebuilt = new OrderedGroupedKVInputConfigurer();
rebuilt.fromUserPayload(configuration.toUserPayload());
Configuration conf = rebuilt.conf;
http://git-wip-us.apache.org/repos/asf/tez/blob/469bf905/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestShuffledUnorderedKVInputConfigurer.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestShuffledUnorderedKVInputConfigurer.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestShuffledUnorderedKVInputConfigurer.java
index 4910bcf..3f980a4 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestShuffledUnorderedKVInputConfigurer.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestShuffledUnorderedKVInputConfigurer.java
@@ -37,14 +37,14 @@ public class TestShuffledUnorderedKVInputConfigurer {
@Test
public void testNullParams() {
try {
- ShuffledUnorderedKVInputConfigurer.newBuilder(null, "VALUE");
+ UnorderedKVInputConfigurer.newBuilder(null, "VALUE");
fail("Expecting a null parameter list to fail");
} catch (NullPointerException npe) {
assertTrue(npe.getMessage().contains("cannot be null"));
}
try {
- ShuffledUnorderedKVInputConfigurer.newBuilder("KEY", null);
+ UnorderedKVInputConfigurer.newBuilder("KEY", null);
fail("Expecting a null parameter list to fail");
} catch (NullPointerException npe) {
assertTrue(npe.getMessage().contains("cannot be null"));
@@ -61,8 +61,8 @@ public class TestShuffledUnorderedKVInputConfigurer {
additionalConf.put("test.key.2", "key2");
additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, "3");
additionalConf.put("file.shouldExist", "file");
- ShuffledUnorderedKVInputConfigurer.Builder builder =
- ShuffledUnorderedKVInputConfigurer.newBuilder("KEY", "VALUE")
+ UnorderedKVInputConfigurer.Builder builder =
+ UnorderedKVInputConfigurer.newBuilder("KEY", "VALUE")
.setCompression(true, "CustomCodec")
.setMaxSingleMemorySegmentFraction(0.11f)
.setMergeFraction(0.22f)
@@ -74,9 +74,9 @@ public class TestShuffledUnorderedKVInputConfigurer {
.setAdditionalConfiguration(additionalConf)
.setFromConfiguration(fromConf);
- ShuffledUnorderedKVInputConfigurer configuration = builder.build();
+ UnorderedKVInputConfigurer configuration = builder.build();
- ShuffledUnorderedKVInputConfigurer rebuilt = new ShuffledUnorderedKVInputConfigurer();
+ UnorderedKVInputConfigurer rebuilt = new UnorderedKVInputConfigurer();
rebuilt.fromUserPayload(configuration.toUserPayload());
Configuration conf = rebuilt.conf;
@@ -108,11 +108,11 @@ public class TestShuffledUnorderedKVInputConfigurer {
@Test
public void testDefaultConfigsUsed() {
- ShuffledUnorderedKVInputConfigurer.Builder builder =
- ShuffledUnorderedKVInputConfigurer.newBuilder("KEY", "VALUE");
- ShuffledUnorderedKVInputConfigurer configuration = builder.build();
+ UnorderedKVInputConfigurer.Builder builder =
+ UnorderedKVInputConfigurer.newBuilder("KEY", "VALUE");
+ UnorderedKVInputConfigurer configuration = builder.build();
- ShuffledUnorderedKVInputConfigurer rebuilt = new ShuffledUnorderedKVInputConfigurer();
+ UnorderedKVInputConfigurer rebuilt = new UnorderedKVInputConfigurer();
rebuilt.fromUserPayload(configuration.toUserPayload());
Configuration conf = rebuilt.conf;
http://git-wip-us.apache.org/repos/asf/tez/blob/469bf905/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfigurer.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfigurer.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfigurer.java
index 1943105..03b218e 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfigurer.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfigurer.java
@@ -68,11 +68,11 @@ public class TestUnorderedPartitionedKVEdgeConfigurer {
UnorderedPartitionedKVEdgeConfigurer configuration = builder.build();
- OnFileUnorderedPartitionedKVOutputConfigurer rebuiltOutput =
- new OnFileUnorderedPartitionedKVOutputConfigurer();
+ UnorderedPartitionedKVOutputConfigurer rebuiltOutput =
+ new UnorderedPartitionedKVOutputConfigurer();
rebuiltOutput.fromUserPayload(configuration.getOutputPayload());
- ShuffledUnorderedKVInputConfigurer rebuiltInput =
- new ShuffledUnorderedKVInputConfigurer();
+ UnorderedKVInputConfigurer rebuiltInput =
+ new UnorderedKVInputConfigurer();
rebuiltInput.fromUserPayload(configuration.getInputPayload());
Configuration outputConf = rebuiltOutput.conf;
@@ -100,11 +100,11 @@ public class TestUnorderedPartitionedKVEdgeConfigurer {
UnorderedPartitionedKVEdgeConfigurer configuration = builder.build();
- OnFileUnorderedPartitionedKVOutputConfigurer rebuiltOutput =
- new OnFileUnorderedPartitionedKVOutputConfigurer();
+ UnorderedPartitionedKVOutputConfigurer rebuiltOutput =
+ new UnorderedPartitionedKVOutputConfigurer();
rebuiltOutput.fromUserPayload(configuration.getOutputPayload());
- ShuffledUnorderedKVInputConfigurer rebuiltInput =
- new ShuffledUnorderedKVInputConfigurer();
+ UnorderedKVInputConfigurer rebuiltInput =
+ new UnorderedKVInputConfigurer();
rebuiltInput.fromUserPayload(configuration.getInputPayload());
Configuration outputConf = rebuiltOutput.conf;
@@ -144,11 +144,11 @@ public class TestUnorderedPartitionedKVEdgeConfigurer {
UnorderedPartitionedKVEdgeConfigurer configuration = builder.build();
- OnFileUnorderedPartitionedKVOutputConfigurer rebuiltOutput =
- new OnFileUnorderedPartitionedKVOutputConfigurer();
+ UnorderedPartitionedKVOutputConfigurer rebuiltOutput =
+ new UnorderedPartitionedKVOutputConfigurer();
rebuiltOutput.fromUserPayload(configuration.getOutputPayload());
- ShuffledUnorderedKVInputConfigurer rebuiltInput =
- new ShuffledUnorderedKVInputConfigurer();
+ UnorderedKVInputConfigurer rebuiltInput =
+ new UnorderedKVInputConfigurer();
rebuiltInput.fromUserPayload(configuration.getInputPayload());
Configuration outputConf = rebuiltOutput.conf;
http://git-wip-us.apache.org/repos/asf/tez/blob/469bf905/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedUnpartitionedKVEdgeConfigurer.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedUnpartitionedKVEdgeConfigurer.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedUnpartitionedKVEdgeConfigurer.java
index c5fdbda..caa2364 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedUnpartitionedKVEdgeConfigurer.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedUnpartitionedKVEdgeConfigurer.java
@@ -38,14 +38,14 @@ public class TestUnorderedUnpartitionedKVEdgeConfigurer {
@Test
public void testNullParams() {
try {
- UnorderedUnpartitionedKVEdgeConfigurer.newBuilder(null, "VALUE");
+ UnorderedKVEdgeConfigurer.newBuilder(null, "VALUE");
fail("Expecting a null parameter list to fail");
} catch (NullPointerException npe) {
assertTrue(npe.getMessage().contains("cannot be null"));
}
try {
- UnorderedUnpartitionedKVEdgeConfigurer.newBuilder("KEY", null);
+ UnorderedKVEdgeConfigurer.newBuilder("KEY", null);
fail("Expecting a null parameter list to fail");
} catch (NullPointerException npe) {
assertTrue(npe.getMessage().contains("cannot be null"));
@@ -54,18 +54,18 @@ public class TestUnorderedUnpartitionedKVEdgeConfigurer {
@Test
public void testDefaultConfigsUsed() {
- UnorderedUnpartitionedKVEdgeConfigurer.Builder builder =
- UnorderedUnpartitionedKVEdgeConfigurer.newBuilder("KEY", "VALUE");
+ UnorderedKVEdgeConfigurer.Builder builder =
+ UnorderedKVEdgeConfigurer.newBuilder("KEY", "VALUE");
builder.setKeySerializationClass("SerClass1").setValueSerializationClass("SerClass2");
- UnorderedUnpartitionedKVEdgeConfigurer configuration = builder.build();
+ UnorderedKVEdgeConfigurer configuration = builder.build();
- OnFileUnorderedKVOutputConfigurer rebuiltOutput =
- new OnFileUnorderedKVOutputConfigurer();
+ UnorderedKVOutputConfigurer rebuiltOutput =
+ new UnorderedKVOutputConfigurer();
rebuiltOutput.fromUserPayload(configuration.getOutputPayload());
- ShuffledUnorderedKVInputConfigurer rebuiltInput =
- new ShuffledUnorderedKVInputConfigurer();
+ UnorderedKVInputConfigurer rebuiltInput =
+ new UnorderedKVInputConfigurer();
rebuiltInput.fromUserPayload(configuration.getInputPayload());
Configuration outputConf = rebuiltOutput.conf;
@@ -88,16 +88,16 @@ public class TestUnorderedUnpartitionedKVEdgeConfigurer {
@Test
public void testSpecificIOConfs() {
// Ensures that Output and Input confs are not mixed.
- UnorderedUnpartitionedKVEdgeConfigurer.Builder builder =
- UnorderedUnpartitionedKVEdgeConfigurer.newBuilder("KEY", "VALUE");
+ UnorderedKVEdgeConfigurer.Builder builder =
+ UnorderedKVEdgeConfigurer.newBuilder("KEY", "VALUE");
- UnorderedUnpartitionedKVEdgeConfigurer configuration = builder.build();
+ UnorderedKVEdgeConfigurer configuration = builder.build();
- OnFileUnorderedKVOutputConfigurer rebuiltOutput =
- new OnFileUnorderedKVOutputConfigurer();
+ UnorderedKVOutputConfigurer rebuiltOutput =
+ new UnorderedKVOutputConfigurer();
rebuiltOutput.fromUserPayload(configuration.getOutputPayload());
- ShuffledUnorderedKVInputConfigurer rebuiltInput =
- new ShuffledUnorderedKVInputConfigurer();
+ UnorderedKVInputConfigurer rebuiltInput =
+ new UnorderedKVInputConfigurer();
rebuiltInput.fromUserPayload(configuration.getInputPayload());
Configuration outputConf = rebuiltOutput.conf;
@@ -123,7 +123,7 @@ public class TestUnorderedUnpartitionedKVEdgeConfigurer {
additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, "0.22f");
additionalConfs.put("file.shouldExist", "file");
- UnorderedUnpartitionedKVEdgeConfigurer.Builder builder = UnorderedUnpartitionedKVEdgeConfigurer
+ UnorderedKVEdgeConfigurer.Builder builder = UnorderedKVEdgeConfigurer
.newBuilder("KEY",
"VALUE")
.setAdditionalConfiguration("fs.shouldExist", "fs")
@@ -133,13 +133,13 @@ public class TestUnorderedUnpartitionedKVEdgeConfigurer {
.setAdditionalConfiguration(additionalConfs)
.setFromConfiguration(fromConf);
- UnorderedUnpartitionedKVEdgeConfigurer configuration = builder.build();
+ UnorderedKVEdgeConfigurer configuration = builder.build();
- OnFileUnorderedKVOutputConfigurer rebuiltOutput =
- new OnFileUnorderedKVOutputConfigurer();
+ UnorderedKVOutputConfigurer rebuiltOutput =
+ new UnorderedKVOutputConfigurer();
rebuiltOutput.fromUserPayload(configuration.getOutputPayload());
- ShuffledUnorderedKVInputConfigurer rebuiltInput =
- new ShuffledUnorderedKVInputConfigurer();
+ UnorderedKVInputConfigurer rebuiltInput =
+ new UnorderedKVInputConfigurer();
rebuiltInput.fromUserPayload(configuration.getInputPayload());
Configuration outputConf = rebuiltOutput.conf;
http://git-wip-us.apache.org/repos/asf/tez/blob/469bf905/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/input/TestSortedGroupedMergedInput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/input/TestSortedGroupedMergedInput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/input/TestSortedGroupedMergedInput.java
index 0816e94..cde0bd2 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/input/TestSortedGroupedMergedInput.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/input/TestSortedGroupedMergedInput.java
@@ -65,7 +65,7 @@ public class TestSortedGroupedMergedInput {
sInputs.add(sInput1);
sInputs.add(sInput2);
sInputs.add(sInput3);
- SortedGroupedMergedInput input = new SortedGroupedMergedInput(createMergedInputContext(), sInputs);
+ OrderedGroupedMergedKVInput input = new OrderedGroupedMergedKVInput(createMergedInputContext(), sInputs);
KeyValuesReader kvsReader = input.getReader();
int keyCount = 0;
@@ -106,7 +106,7 @@ public class TestSortedGroupedMergedInput {
sInputs.add(sInput2);
sInputs.add(sInput3);
- SortedGroupedMergedInput input = new SortedGroupedMergedInput(createMergedInputContext(), sInputs);
+ OrderedGroupedMergedKVInput input = new OrderedGroupedMergedKVInput(createMergedInputContext(), sInputs);
KeyValuesReader kvsReader = input.getReader();
int keyCount = 0;
@@ -149,7 +149,7 @@ public class TestSortedGroupedMergedInput {
sInputs.add(sInput2);
sInputs.add(sInput3);
- SortedGroupedMergedInput input = new SortedGroupedMergedInput(createMergedInputContext(), sInputs);
+ OrderedGroupedMergedKVInput input = new OrderedGroupedMergedKVInput(createMergedInputContext(), sInputs);
KeyValuesReader kvsReader = input.getReader();
int keyCount = 0;
while (kvsReader.next()) {
@@ -195,7 +195,7 @@ public class TestSortedGroupedMergedInput {
sInputs.add(sInput2);
sInputs.add(sInput3);
- SortedGroupedMergedInput input = new SortedGroupedMergedInput(createMergedInputContext(), sInputs);
+ OrderedGroupedMergedKVInput input = new OrderedGroupedMergedKVInput(createMergedInputContext(), sInputs);
KeyValuesReader kvsReader = input.getReader();
int keyCount = 0;
while (kvsReader.next()) {
@@ -246,7 +246,7 @@ public class TestSortedGroupedMergedInput {
sInputs.add(sInput2);
sInputs.add(sInput3);
- SortedGroupedMergedInput input = new SortedGroupedMergedInput(createMergedInputContext(), sInputs);
+ OrderedGroupedMergedKVInput input = new OrderedGroupedMergedKVInput(createMergedInputContext(), sInputs);
KeyValuesReader kvsReader = input.getReader();
int keyCount = 0;
while (kvsReader.next()) {
@@ -301,7 +301,7 @@ public class TestSortedGroupedMergedInput {
sInputs.add(sInput2);
sInputs.add(sInput3);
- SortedGroupedMergedInput input = new SortedGroupedMergedInput(createMergedInputContext(), sInputs);
+ OrderedGroupedMergedKVInput input = new OrderedGroupedMergedKVInput(createMergedInputContext(), sInputs);
KeyValuesReader kvsReader = input.getReader();
int keyCount = 0;
@@ -347,13 +347,13 @@ public class TestSortedGroupedMergedInput {
sInputs.add(sInput2);
sInputs.add(sInput3);
- SortedGroupedMergedInput input = new SortedGroupedMergedInput(createMergedInputContext(), sInputs);
+ OrderedGroupedMergedKVInput input = new OrderedGroupedMergedKVInput(createMergedInputContext(), sInputs);
KeyValuesReader kvsReader = input.getReader();
assertFalse(kvsReader.next());
}
- private static class SortedTestInput extends ShuffledMergedInput {
+ private static class SortedTestInput extends OrderedGroupedKVInput {
final SortedTestKeyValuesReader reader;
http://git-wip-us.apache.org/repos/asf/tez/blob/469bf905/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
index 3b58ad3..e49e336 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
@@ -81,7 +81,7 @@ public class TestOnFileSortedOutput {
private int sorterThreads;
private KeyValuesWriter writer;
- private OnFileSortedOutput sortedOutput;
+ private OrderedPartitionedKVOutput sortedOutput;
private boolean sendEmptyPartitionViaEvent;
//Partition index for which data should not be written to.
private int emptyPartitionIdx;
@@ -150,7 +150,7 @@ public class TestOnFileSortedOutput {
private void startSortedOutput(int partitions) throws Exception {
OutputContext context = createTezOutputContext();
- sortedOutput = new OnFileSortedOutput(context, partitions);
+ sortedOutput = new OrderedPartitionedKVOutput(context, partitions);
sortedOutput.initialize();
sortedOutput.start();
writer = sortedOutput.getWriter();
http://git-wip-us.apache.org/repos/asf/tez/blob/469bf905/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java
index 7c4b605..1c6db22 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java
@@ -125,7 +125,7 @@ public class TestOnFileUnorderedKVOutput {
taskAttemptID, counters, 0, userPayload, runtimeTask,
null, auxEnv, new MemoryDistributor(1, 1, conf) , outputDescriptor, null);
- OnFileUnorderedKVOutput kvOutput = new OnFileUnorderedKVOutputForTest(outputContext, 1);
+ UnorderedKVOutput kvOutput = new OnFileUnorderedKVOutputForTest(outputContext, 1);
List<Event> events = null;
@@ -153,7 +153,7 @@ public class TestOnFileUnorderedKVOutput {
assertEquals("host", shufflePayload.getHost());
}
- private static class OnFileUnorderedKVOutputForTest extends OnFileUnorderedKVOutput {
+ private static class OnFileUnorderedKVOutputForTest extends UnorderedKVOutput {
public OnFileUnorderedKVOutputForTest(OutputContext outputContext, int numPhysicalOutputs) {
super(outputContext, numPhysicalOutputs);
http://git-wip-us.apache.org/repos/asf/tez/blob/469bf905/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
index 35a7c76..1eff69c 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
@@ -49,8 +49,8 @@ import org.apache.tez.runtime.api.ObjectRegistry;
import org.apache.tez.runtime.api.ProcessorContext;
import org.apache.tez.runtime.library.api.KeyValueReader;
import org.apache.tez.runtime.library.api.KeyValueWriter;
-import org.apache.tez.runtime.library.conf.UnorderedUnpartitionedKVEdgeConfigurer;
-import org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput;
+import org.apache.tez.runtime.library.conf.UnorderedKVEdgeConfigurer;
+import org.apache.tez.runtime.library.output.UnorderedKVOutput;
import org.apache.tez.runtime.library.processor.SimpleProcessor;
import com.google.common.base.Preconditions;
@@ -66,7 +66,7 @@ public class BroadcastAndOneToOneExample extends Configured implements Tool {
@Override
public void run() throws Exception {
Preconditions.checkArgument(getOutputs().size() == 1);
- OnFileUnorderedKVOutput output = (OnFileUnorderedKVOutput) getOutputs().values().iterator()
+ UnorderedKVOutput output = (UnorderedKVOutput) getOutputs().values().iterator()
.next();
KeyValueWriter kvWriter = (KeyValueWriter) output.getWriter();
kvWriter.write(word, new IntWritable(getContext().getTaskIndex()));
@@ -157,7 +157,7 @@ public class BroadcastAndOneToOneExample extends Configured implements Tool {
oneToOneVertex.setVertexManagerPlugin(
new VertexManagerPluginDescriptor(InputReadyVertexManager.class.getName()));
- UnorderedUnpartitionedKVEdgeConfigurer edgeConf = UnorderedUnpartitionedKVEdgeConfigurer
+ UnorderedKVEdgeConfigurer edgeConf = UnorderedKVEdgeConfigurer
.newBuilder(Text.class.getName(), IntWritable.class.getName()).build();
DAG dag = new DAG("BroadcastAndOneToOneExample");
http://git-wip-us.apache.org/repos/asf/tez/blob/469bf905/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
index 02a3202..54131e7 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
@@ -75,7 +75,7 @@ import org.apache.tez.mapreduce.input.MRInputLegacy;
import org.apache.tez.mapreduce.output.MROutput;
import org.apache.tez.processor.FilterByWordInputProcessor;
import org.apache.tez.processor.FilterByWordOutputProcessor;
-import org.apache.tez.runtime.library.conf.UnorderedUnpartitionedKVEdgeConfigurer;
+import org.apache.tez.runtime.library.conf.UnorderedKVEdgeConfigurer;
import com.google.common.collect.Sets;
@@ -197,7 +197,7 @@ public class FilterLinesByWord extends Configured implements Tool {
OutputCommitterDescriptor ocd = new OutputCommitterDescriptor(MROutputCommitter.class.getName());
stage2Vertex.addDataSink("MROutput", new DataSinkDescriptor(od, ocd, null));
- UnorderedUnpartitionedKVEdgeConfigurer edgeConf = UnorderedUnpartitionedKVEdgeConfigurer
+ UnorderedKVEdgeConfigurer edgeConf = UnorderedKVEdgeConfigurer
.newBuilder(Text.class.getName(), TextLongPair.class.getName()).build();
DAG dag = new DAG("FilterLinesByWord");
http://git-wip-us.apache.org/repos/asf/tez/blob/469bf905/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
index b413993..9bbde1c 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
@@ -68,7 +68,7 @@ import org.apache.tez.mapreduce.input.MRInputLegacy;
import org.apache.tez.mapreduce.output.MROutput;
import org.apache.tez.processor.FilterByWordInputProcessor;
import org.apache.tez.processor.FilterByWordOutputProcessor;
-import org.apache.tez.runtime.library.conf.UnorderedUnpartitionedKVEdgeConfigurer;
+import org.apache.tez.runtime.library.conf.UnorderedKVEdgeConfigurer;
public class FilterLinesByWordOneToOne extends Configured implements Tool {
@@ -189,7 +189,7 @@ public class FilterLinesByWordOneToOne extends Configured implements Tool {
.setUserPayload(TezUtils.createUserPayloadFromConf(stage2Conf)),
new OutputCommitterDescriptor(MROutputCommitter.class.getName()), null));
- UnorderedUnpartitionedKVEdgeConfigurer edgeConf = UnorderedUnpartitionedKVEdgeConfigurer
+ UnorderedKVEdgeConfigurer edgeConf = UnorderedKVEdgeConfigurer
.newBuilder(Text.class.getName(), TextLongPair.class.getName()).build();
DAG dag = new DAG("FilterLinesByWord");