You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2013/09/23 19:45:29 UTC
[15/20] TEZ-443. Merge tez-dag-api and tez-engine-api into a single
module - tez-api (part of TEZ-398). (sseth)
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-common/src/main/java/org/apache/tez/common/counters/GenericCounter.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/counters/GenericCounter.java b/tez-common/src/main/java/org/apache/tez/common/counters/GenericCounter.java
deleted file mode 100644
index 5477606..0000000
--- a/tez-common/src/main/java/org/apache/tez/common/counters/GenericCounter.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.common.counters;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableUtils;
-
-/**
- * A generic counter implementation
- */
-@InterfaceAudience.Private
-public class GenericCounter extends AbstractCounter {
-
- private String name;
- private String displayName;
- private long value = 0;
-
- public GenericCounter() {
- // mostly for readFields
- }
-
- public GenericCounter(String name, String displayName) {
- this.name = name;
- this.displayName = displayName;
- }
-
- public GenericCounter(String name, String displayName, long value) {
- this.name = name;
- this.displayName = displayName;
- this.value = value;
- }
-
- @Override @Deprecated
- public synchronized void setDisplayName(String displayName) {
- this.displayName = displayName;
- }
-
- @Override
- public synchronized void readFields(DataInput in) throws IOException {
- name = Text.readString(in);
- displayName = in.readBoolean() ? Text.readString(in) : name;
- value = WritableUtils.readVLong(in);
- }
-
- /**
- * GenericCounter ::= keyName isDistinctDisplayName [displayName] value
- */
- @Override
- public synchronized void write(DataOutput out) throws IOException {
- Text.writeString(out, name);
- boolean distinctDisplayName = ! name.equals(displayName);
- out.writeBoolean(distinctDisplayName);
- if (distinctDisplayName) {
- Text.writeString(out, displayName);
- }
- WritableUtils.writeVLong(out, value);
- }
-
- @Override
- public synchronized String getName() {
- return name;
- }
-
- @Override
- public synchronized String getDisplayName() {
- return displayName;
- }
-
- @Override
- public synchronized long getValue() {
- return value;
- }
-
- @Override
- public synchronized void setValue(long value) {
- this.value = value;
- }
-
- @Override
- public synchronized void increment(long incr) {
- value += incr;
- }
-
- @Override
- public TezCounter getUnderlyingCounter() {
- return this;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-common/src/main/java/org/apache/tez/common/counters/JobCounter.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/counters/JobCounter.java b/tez-common/src/main/java/org/apache/tez/common/counters/JobCounter.java
deleted file mode 100644
index 1eb2be8..0000000
--- a/tez-common/src/main/java/org/apache/tez/common/counters/JobCounter.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.common.counters;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-// Per-job counters
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public enum JobCounter {
- NUM_FAILED_MAPS,
- NUM_FAILED_REDUCES,
- NUM_KILLED_MAPS,
- NUM_KILLED_REDUCES,
- TOTAL_LAUNCHED_MAPS,
- TOTAL_LAUNCHED_REDUCES,
- OTHER_LOCAL_MAPS,
- DATA_LOCAL_MAPS,
- RACK_LOCAL_MAPS,
- SLOTS_MILLIS_MAPS,
- SLOTS_MILLIS_REDUCES,
- FALLOW_SLOTS_MILLIS_MAPS,
- FALLOW_SLOTS_MILLIS_REDUCES,
- TOTAL_LAUNCHED_UBERTASKS,
- NUM_UBER_SUBMAPS,
- NUM_UBER_SUBREDUCES,
- NUM_FAILED_UBERTASKS
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-common/src/main/java/org/apache/tez/common/counters/LimitExceededException.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/counters/LimitExceededException.java b/tez-common/src/main/java/org/apache/tez/common/counters/LimitExceededException.java
deleted file mode 100644
index e50bd81..0000000
--- a/tez-common/src/main/java/org/apache/tez/common/counters/LimitExceededException.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.common.counters;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-@InterfaceAudience.Private
-public class LimitExceededException extends RuntimeException {
-
- private static final long serialVersionUID = 1L;
-
- public LimitExceededException(String msg) {
- super(msg);
- }
-
- // Only allows chaining of related exceptions
- public LimitExceededException(LimitExceededException cause) {
- super(cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-common/src/main/java/org/apache/tez/common/counters/Limits.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/counters/Limits.java b/tez-common/src/main/java/org/apache/tez/common/counters/Limits.java
deleted file mode 100644
index aacce87..0000000
--- a/tez-common/src/main/java/org/apache/tez/common/counters/Limits.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.common.counters;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.tez.common.TezJobConfig;
-
-@InterfaceAudience.Private
-public class Limits {
-
- private static Configuration conf = null;
- private static int GROUP_NAME_MAX;
- private static int COUNTER_NAME_MAX;
- private static int GROUPS_MAX;
- private static int COUNTERS_MAX;
- private static boolean initialized = false;
-
- private static synchronized void ensureInitialized() {
- if (initialized) {
- return;
- }
- if (conf == null) {
- conf = new Configuration();
- }
- GROUP_NAME_MAX =
- conf.getInt(TezJobConfig.COUNTER_GROUP_NAME_MAX_KEY,
- TezJobConfig.COUNTER_GROUP_NAME_MAX_DEFAULT);
- COUNTER_NAME_MAX =
- conf.getInt(TezJobConfig.COUNTER_NAME_MAX_KEY,
- TezJobConfig.COUNTER_NAME_MAX_DEFAULT);
- GROUPS_MAX =
- conf.getInt(TezJobConfig.COUNTER_GROUPS_MAX_KEY,
- TezJobConfig.COUNTER_GROUPS_MAX_DEFAULT);
- COUNTERS_MAX =
- conf.getInt(TezJobConfig.COUNTERS_MAX_KEY, TezJobConfig.
- COUNTERS_MAX_DEFAULT);
- initialized = true;
- }
-
- private int totalCounters;
- private LimitExceededException firstViolation;
-
- public static String filterName(String name, int maxLen) {
- return name.length() > maxLen ? name.substring(0, maxLen - 1) : name;
- }
-
- public static String filterCounterName(String name) {
- ensureInitialized();
- return filterName(name, COUNTER_NAME_MAX);
- }
-
- public static String filterGroupName(String name) {
- ensureInitialized();
- return filterName(name, GROUP_NAME_MAX);
- }
-
- public synchronized void checkCounters(int size) {
- ensureInitialized();
- if (firstViolation != null) {
- throw new LimitExceededException(firstViolation);
- }
- if (size > COUNTERS_MAX) {
- firstViolation = new LimitExceededException("Too many counters: "+ size +
- " max="+ COUNTERS_MAX);
- throw firstViolation;
- }
- }
-
- public synchronized void incrCounters() {
- checkCounters(totalCounters + 1);
- ++totalCounters;
- }
-
- public synchronized void checkGroups(int size) {
- ensureInitialized();
- if (firstViolation != null) {
- throw new LimitExceededException(firstViolation);
- }
- if (size > GROUPS_MAX) {
- firstViolation = new LimitExceededException("Too many counter groups: "+
- size +" max="+ GROUPS_MAX);
- }
- }
-
- public synchronized LimitExceededException violation() {
- return firstViolation;
- }
-
- public synchronized static void setConfiguration(Configuration conf) {
- if (Limits.conf == null && conf != null) {
- Limits.conf = conf;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-common/src/main/java/org/apache/tez/common/counters/ResourceBundles.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/counters/ResourceBundles.java b/tez-common/src/main/java/org/apache/tez/common/counters/ResourceBundles.java
deleted file mode 100644
index 8113cab..0000000
--- a/tez-common/src/main/java/org/apache/tez/common/counters/ResourceBundles.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.common.counters;
-
-import java.util.Locale;
-import java.util.ResourceBundle;
-import java.util.MissingResourceException;
-
-/**
- * Helper class to handle resource bundles in a saner way
- */
-public class ResourceBundles {
-
- /**
- * Get a resource bundle
- * @param bundleName of the resource
- * @return the resource bundle
- * @throws MissingResourceException
- */
- public static ResourceBundle getBundle(String bundleName) {
- return ResourceBundle.getBundle(bundleName.replace('$', '_'),
- Locale.getDefault(), Thread.currentThread().getContextClassLoader());
- }
-
- /**
- * Get a resource given bundle name and key
- * @param <T> type of the resource
- * @param bundleName name of the resource bundle
- * @param key to lookup the resource
- * @param suffix for the key to lookup
- * @param defaultValue of the resource
- * @return the resource or the defaultValue
- * @throws ClassCastException if the resource found doesn't match T
- */
- @SuppressWarnings("unchecked")
- public static synchronized <T> T getValue(String bundleName, String key,
- String suffix, T defaultValue) {
- T value;
- try {
- ResourceBundle bundle = getBundle(bundleName);
- value = (T) bundle.getObject(getLookupKey(key, suffix));
- if (value != null) {
- return value;
- }
- }
- catch (Exception e) {
- // Ignore
- }
- return defaultValue;
- }
-
- private static String getLookupKey(String key, String suffix) {
- if (suffix == null || suffix.isEmpty()) return key;
- return key + suffix;
- }
-
- /**
- * Get the counter group display name
- * @param group the group name to lookup
- * @param defaultValue of the group
- * @return the group display name
- */
- public static String getCounterGroupName(String group, String defaultValue) {
- return getValue(group, "CounterGroupName", "", defaultValue);
- }
-
- /**
- * Get the counter display name
- * @param group the counter group name for the counter
- * @param counter the counter name to lookup
- * @param defaultValue of the counter
- * @return the counter display name
- */
- public static String getCounterName(String group, String counter,
- String defaultValue) {
- return getValue(group, counter, ".name", defaultValue);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-common/src/main/java/org/apache/tez/common/counters/TaskCounter.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/counters/TaskCounter.java b/tez-common/src/main/java/org/apache/tez/common/counters/TaskCounter.java
deleted file mode 100644
index b6fca27..0000000
--- a/tez-common/src/main/java/org/apache/tez/common/counters/TaskCounter.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.common.counters;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-// TODO TEZAM5 For MR compatibility, a conversion from tez.TaskCounters to
-// mapreduce.TaskCounters will likely be required somewhere.
-// Similarly for FileSystemCounters and others.
-
-// Counters used by Task classes
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public enum TaskCounter {
- // TODO Eventually, rename counters to be non-MR specific and map them to MR equivalent.
- MAP_INPUT_RECORDS,
- MAP_OUTPUT_RECORDS,
- MAP_SKIPPED_RECORDS,
- MAP_OUTPUT_BYTES,
- MAP_OUTPUT_MATERIALIZED_BYTES,
- SPLIT_RAW_BYTES,
- COMBINE_INPUT_RECORDS,
- COMBINE_OUTPUT_RECORDS,
- REDUCE_INPUT_GROUPS,
- REDUCE_SHUFFLE_BYTES,
- REDUCE_INPUT_RECORDS,
- REDUCE_OUTPUT_RECORDS,
- REDUCE_SKIPPED_GROUPS,
- REDUCE_SKIPPED_RECORDS,
- SPILLED_RECORDS,
- SHUFFLED_MAPS,
- FAILED_SHUFFLE,
- MERGED_MAP_OUTPUTS,
- GC_TIME_MILLIS,
- CPU_MILLISECONDS,
- PHYSICAL_MEMORY_BYTES,
- VIRTUAL_MEMORY_BYTES,
- COMMITTED_HEAP_BYTES,
-
- INPUT_RECORDS,
- OUTPUT_RECORDS,
- SKIPPED_RECORDS,
- OUTPUT_BYTES,
- OUTPUT_MATERIALIZED_BYTES,
- INPUT_GROUPS,
- SHUFFLE_BYTES,
- SHUFFLED_TASKS,
- MERGED_TASK_OUTPUTS,
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-common/src/main/java/org/apache/tez/common/counters/TezCounter.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/counters/TezCounter.java b/tez-common/src/main/java/org/apache/tez/common/counters/TezCounter.java
deleted file mode 100644
index 394c820..0000000
--- a/tez-common/src/main/java/org/apache/tez/common/counters/TezCounter.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.common.counters;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.io.Writable;
-
-/**
- * A named counter that tracks the progress of a map/reduce job.
- *
- * <p><code>Counters</code> represent global counters, defined either by the
- * Map-Reduce framework or applications. Each <code>Counter</code> is named by
- * an {@link Enum} and has a long for the value.</p>
- *
- * <p><code>Counters</code> are bunched into Groups, each comprising of
- * counters from a particular <code>Enum</code> class.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public interface TezCounter extends Writable {
-
- /**
- * Set the display name of the counter
- * @param displayName of the counter
- * @deprecated (and no-op by default)
- */
- @Deprecated
- void setDisplayName(String displayName);
-
- /**
- * @return the name of the counter
- */
- String getName();
-
- /**
- * Get the display name of the counter.
- * @return the user facing name of the counter
- */
- String getDisplayName();
-
- /**
- * What is the current value of this counter?
- * @return the current value
- */
- long getValue();
-
- /**
- * Set this counter by the given value
- * @param value the value to set
- */
- void setValue(long value);
-
- /**
- * Increment this counter by the given value
- * @param incr the value to increase this counter by
- */
- void increment(long incr);
-
- /**
- * Return the underlying object if this is a facade.
- * @return the undelying object.
- */
- @Private
- TezCounter getUnderlyingCounter();
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-common/src/main/java/org/apache/tez/common/counters/TezCounters.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/counters/TezCounters.java b/tez-common/src/main/java/org/apache/tez/common/counters/TezCounters.java
deleted file mode 100644
index 1c9521a..0000000
--- a/tez-common/src/main/java/org/apache/tez/common/counters/TezCounters.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tez.common.counters;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * <p><code>Counters</code> holds per job/task counters, defined either by the
- * Map-Reduce framework or applications. Each <code>Counter</code> can be of
- * any {@link Enum} type.</p>
- *
- * <p><code>Counters</code> are bunched into {@link CounterGroup}s, each
- * comprising of counters from a particular <code>Enum</code> class.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public class TezCounters extends AbstractCounters<TezCounter, CounterGroup> {
-
- // Mix framework group implementation into CounterGroup interface
- private static class FrameworkGroupImpl<T extends Enum<T>>
- extends FrameworkCounterGroup<T, TezCounter> implements CounterGroup {
-
- FrameworkGroupImpl(Class<T> cls) {
- super(cls);
- }
-
- @Override
- protected FrameworkCounter<T> newCounter(T key) {
- return new FrameworkCounter<T>(key, getName());
- }
-
- @Override
- public CounterGroupBase<TezCounter> getUnderlyingGroup() {
- return this;
- }
- }
-
- // Mix generic group implementation into CounterGroup interface
- // and provide some mandatory group factory methods.
- private static class GenericGroup extends AbstractCounterGroup<TezCounter>
- implements CounterGroup {
-
- GenericGroup(String name, String displayName, Limits limits) {
- super(name, displayName, limits);
- }
-
- @Override
- protected TezCounter newCounter(String name, String displayName, long value) {
- return new GenericCounter(name, displayName, value);
- }
-
- @Override
- protected TezCounter newCounter() {
- return new GenericCounter();
- }
-
- @Override
- public CounterGroupBase<TezCounter> getUnderlyingGroup() {
- return this;
- }
- }
-
- // Mix file system group implementation into the CounterGroup interface
- private static class FileSystemGroup extends FileSystemCounterGroup<TezCounter>
- implements CounterGroup {
-
- @Override
- protected TezCounter newCounter(String scheme, FileSystemCounter key) {
- return new FSCounter(scheme, key);
- }
-
- @Override
- public CounterGroupBase<TezCounter> getUnderlyingGroup() {
- return this;
- }
- }
-
- /**
- * Provide factory methods for counter group factory implementation.
- * See also the GroupFactory in
- * {@link org.apache.hadoop.TezCounters.Counters mapred.Counters}
- */
- private static class GroupFactory
- extends CounterGroupFactory<TezCounter, CounterGroup> {
-
- @Override
- protected <T extends Enum<T>>
- FrameworkGroupFactory<CounterGroup>
- newFrameworkGroupFactory(final Class<T> cls) {
- return new FrameworkGroupFactory<CounterGroup>() {
- @Override public CounterGroup newGroup(String name) {
- return new FrameworkGroupImpl<T>(cls); // impl in this package
- }
- };
- }
-
- @Override
- protected CounterGroup newGenericGroup(String name, String displayName,
- Limits limits) {
- return new GenericGroup(name, displayName, limits);
- }
-
- @Override
- protected CounterGroup newFileSystemGroup() {
- return new FileSystemGroup();
- }
- }
-
- private static final GroupFactory groupFactory = new GroupFactory();
-
- /**
- * Default constructor
- */
- public TezCounters() {
- super(groupFactory);
- }
-
- /**
- * Construct the Counters object from the another counters object
- * @param <C> the type of counter
- * @param <G> the type of counter group
- * @param counters the old counters object
- */
- public <C extends TezCounter, G extends CounterGroupBase<C>>
- TezCounters(AbstractCounters<C, G> counters) {
- super(counters, groupFactory);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java b/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java
deleted file mode 100644
index 193b979..0000000
--- a/tez-common/src/main/java/org/apache/tez/records/TezContainerId.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.tez.records;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-
-//TODO EVENTUALLY Once everything is on PB, get rid of this.
-//Alternately have the PB interfaces implement Writable.
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class TezContainerId implements Writable {
-
- private ContainerId containerId;
-
- public TezContainerId() {
- }
-
- public TezContainerId(ContainerId containerId) {
- this.containerId = containerId;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeLong(containerId.getApplicationAttemptId().getApplicationId()
- .getClusterTimestamp());
- out.writeInt(containerId.getApplicationAttemptId().getApplicationId()
- .getId());
- out.writeInt(containerId.getApplicationAttemptId().getAttemptId());
- out.writeInt(containerId.getId());
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- long timestamp = in.readLong();
- int appId = in.readInt();
- int appAttemptId = in.readInt();
- int id = in.readInt();
-
- ApplicationId applicationId = ApplicationId.newInstance(timestamp, appId);
- ApplicationAttemptId applicationAttemptId = ApplicationAttemptId
- .newInstance(applicationId, appAttemptId);
-
- this.containerId = ContainerId.newInstance(applicationAttemptId, id);
- }
-
- @Override
- public String toString() {
- return containerId.toString();
- }
-
- public ContainerId getContainerId() {
- return this.containerId;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-dag-api/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/tez-dag-api/findbugs-exclude.xml b/tez-dag-api/findbugs-exclude.xml
deleted file mode 100644
index 5b11308..0000000
--- a/tez-dag-api/findbugs-exclude.xml
+++ /dev/null
@@ -1,16 +0,0 @@
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-<FindBugsFilter>
-
-</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-dag-api/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dag-api/pom.xml b/tez-dag-api/pom.xml
deleted file mode 100644
index 51d6c73..0000000
--- a/tez-dag-api/pom.xml
+++ /dev/null
@@ -1,88 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.tez</groupId>
- <artifactId>tez</artifactId>
- <version>0.2.0-SNAPSHOT</version>
- </parent>
- <artifactId>tez-dag-api</artifactId>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-common</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-maven-plugins</artifactId>
- <executions>
- <execution>
- <id>compile-protoc</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>protoc</goal>
- </goals>
- <configuration>
- <protocVersion>${protobuf.version}</protocVersion>
- <protocCommand>${protoc.path}</protocCommand>
- <imports>
- <param>${basedir}/src/main/proto</param>
- </imports>
- <source>
- <directory>${basedir}/src/main/proto</directory>
- <includes>
- <include>DAGApiRecords.proto</include>
- <include>DAGClientAMProtocol.proto</include>
- </includes>
- </source>
- <output>${project.build.directory}/generated-sources/java</output>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-dag-api/src/main/java/org/apache/tez/client/AMConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/client/AMConfiguration.java b/tez-dag-api/src/main/java/org/apache/tez/client/AMConfiguration.java
deleted file mode 100644
index f452c74..0000000
--- a/tez-dag-api/src/main/java/org/apache/tez/client/AMConfiguration.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.client;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezUncheckedException;
-
-public class AMConfiguration {
-
- private final Path stagingDir;
- private final String queueName;
- private final Map<String, String> env;
- private final Map<String, LocalResource> localResources;
- private final TezConfiguration amConf;
- private final Credentials credentials;
-
- public AMConfiguration(String queueName, Map<String, String> env,
- Map<String, LocalResource> localResources,
- TezConfiguration conf, Credentials credentials) {
- this.queueName = queueName;
- if (conf != null) {
- this.amConf = conf;
- } else {
- this.amConf = new TezConfiguration();
- }
-
- if (env != null) {
- this.env = env;
- } else {
- this.env = new HashMap<String, String>(0);
- }
- this.localResources = localResources;
- String stagingDirStr = amConf.get(TezConfiguration.TEZ_AM_STAGING_DIR);
- if (stagingDirStr == null || stagingDirStr.isEmpty()) {
- throw new TezUncheckedException("Staging directory for AM resources"
- + " not specified in config"
- + ", property=" + TezConfiguration.TEZ_AM_STAGING_DIR);
- }
- try {
- FileSystem fs = FileSystem.get(amConf);
- this.stagingDir = fs.resolvePath(new Path(stagingDirStr));
- } catch (IOException e) {
- throw new TezUncheckedException(e);
- }
- this.credentials = credentials;
- }
-
- public Path getStagingDir() {
- return stagingDir;
- }
-
- public String getQueueName() {
- return queueName;
- }
-
- public Map<String, String> getEnv() {
- return env;
- }
-
- public Map<String, LocalResource> getLocalResources() {
- return localResources;
- }
-
- public TezConfiguration getAMConf() {
- return amConf;
- }
-
- public Credentials getCredentials() {
- return credentials;
- }
-
- public void isCompatible(AMConfiguration other) {
- // TODO implement
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
deleted file mode 100644
index df260ec..0000000
--- a/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.tez.client;
-
-import java.io.IOException;
-import java.text.NumberFormat;
-import java.util.Map;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.client.DAGClient;
-import org.apache.tez.dag.api.client.rpc.DAGClientRPCImpl;
-
-public class TezClient {
- private static final Log LOG = LogFactory.getLog(TezClient.class);
-
- private final TezConfiguration conf;
- private final YarnConfiguration yarnConf;
- private YarnClient yarnClient;
- Map<String, LocalResource> tezJarResources = null;
-
- /**
- * <p>
- * Create an instance of the TezClient which will be used to communicate with
- * a specific instance of YARN, or TezService when that exists.
- * </p>
- * <p>
- * Separate instances of TezClient should be created to communicate with
- * different instances of YARN
- * </p>
- *
- * @param conf
- * the configuration which will be used to establish which YARN or
- * Tez service instance this client is associated with.
- */
- public TezClient(TezConfiguration conf) {
- this.conf = conf;
- this.yarnConf = new YarnConfiguration(conf);
- yarnClient = new YarnClientImpl();
- yarnClient.init(yarnConf);
- yarnClient.start();
- }
-
-
- public DAGClient submitDAGApplication(DAG dag, AMConfiguration amConfig)
- throws TezException, IOException {
- ApplicationId appId = createApplication();
- return submitDAGApplication(appId, dag, amConfig);
- }
-
- @Private
- // To be used only by YarnRunner
- public DAGClient submitDAGApplication(ApplicationId appId,
- DAG dag, AMConfiguration amConfig)
- throws TezException, IOException {
- try {
- ApplicationSubmissionContext appContext =
- TezClientUtils.createApplicationSubmissionContext(conf, appId, dag,
- dag.getName(), amConfig, getTezJarResources());
- LOG.info("Submitting DAG to YARN"
- + ", applicationId=" + appId);
- yarnClient.submitApplication(appContext);
- } catch (YarnException e) {
- throw new TezException(e);
- }
- return getDAGClient(appId);
- }
-
- /**
- * Create a new YARN application
- * @return <code>ApplicationId</code> for the new YARN application
- * @throws YarnException
- * @throws IOException
- */
- public ApplicationId createApplication() throws TezException, IOException {
- try {
- return yarnClient.createApplication().
- getNewApplicationResponse().getApplicationId();
- } catch (YarnException e) {
- throw new TezException(e);
- }
- }
-
- private synchronized Map<String, LocalResource> getTezJarResources()
- throws IOException {
- if (tezJarResources == null) {
- tezJarResources = TezClientUtils.setupTezJarsLocalResources(conf);
- }
- return tezJarResources;
- }
-
- @Private
- public DAGClient getDAGClient(ApplicationId appId)
- throws IOException, TezException {
- return new DAGClientRPCImpl(appId, getDefaultTezDAGID(appId),
- conf);
- }
-
- // DO NOT CHANGE THIS. This code is replicated from TezDAGID.java
- private static final char SEPARATOR = '_';
- private static final String DAG = "dag";
- private static final NumberFormat idFormat = NumberFormat.getInstance();
- static {
- idFormat.setGroupingUsed(false);
- idFormat.setMinimumIntegerDigits(6);
- }
-
- String getDefaultTezDAGID(ApplicationId appId) {
- return (new StringBuilder(DAG)).append(SEPARATOR).
- append(appId.getClusterTimestamp()).
- append(SEPARATOR).
- append(appId.getId()).
- append(SEPARATOR).
- append(idFormat.format(1)).toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-dag-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-dag-api/src/main/java/org/apache/tez/client/TezClientUtils.java
deleted file mode 100644
index 7c6a5ed..0000000
--- a/tez-dag-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ /dev/null
@@ -1,560 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.client;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.Vector;
-import java.util.Map.Entry;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.Apps;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.log4j.Level;
-import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezConstants;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.api.Vertex;
-import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolBlockingPB;
-import org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto;
-import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
-import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair;
-
-import com.google.common.annotations.VisibleForTesting;
-
-public class TezClientUtils {
-
- private static Log LOG = LogFactory.getLog(TezClientUtils.class);
-
- public static final FsPermission TEZ_AM_DIR_PERMISSION =
- FsPermission.createImmutable((short) 0700); // rwx--------
- public static final FsPermission TEZ_AM_FILE_PERMISSION =
- FsPermission.createImmutable((short) 0644); // rw-r--r--
-
- private static final int UTF8_CHUNK_SIZE = 16 * 1024;
-
- /**
- * Setup LocalResource map for Tez jars based on provided Configuration
- * @param conf Configuration to use to access Tez jars' locations
- * @return Map of LocalResources to use when launching Tez AM
- * @throws IOException
- */
- static Map<String, LocalResource> setupTezJarsLocalResources(
- TezConfiguration conf)
- throws IOException {
- Map<String, LocalResource> tezJarResources =
- new TreeMap<String, LocalResource>();
- if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
- return tezJarResources;
- }
-
- // Add tez jars to local resource
- String[] tezJarUris = conf.getStrings(
- TezConfiguration.TEZ_LIB_URIS);
- if (tezJarUris == null
- || tezJarUris.length == 0) {
- throw new TezUncheckedException("Invalid configuration of tez jars"
- + ", " + TezConfiguration.TEZ_LIB_URIS
- + " is not defined in the configurartion");
- }
-
- for (String tezJarUri : tezJarUris) {
- URI uri;
- try {
- uri = new URI(tezJarUri.trim());
- } catch (URISyntaxException e) {
- String message = "Invalid URI defined in configuration for"
- + " location of TEZ jars. providedURI=" + tezJarUri;
- LOG.error(message);
- throw new TezUncheckedException(message, e);
- }
- if (!uri.isAbsolute()) {
- String message = "Non-absolute URI defined in configuration for"
- + " location of TEZ jars. providedURI=" + tezJarUri;
- LOG.error(message);
- throw new TezUncheckedException(message);
- }
- Path p = new Path(uri);
- FileSystem pathfs = p.getFileSystem(conf);
- RemoteIterator<LocatedFileStatus> iter = pathfs.listFiles(p, false);
- while (iter.hasNext()) {
- LocatedFileStatus fStatus = iter.next();
- String rsrcName = fStatus.getPath().getName();
- // FIXME currently not checking for duplicates due to quirks
- // in assembly generation
- if (tezJarResources.containsKey(rsrcName)) {
- String message = "Duplicate resource found"
- + ", resourceName=" + rsrcName
- + ", existingPath=" +
- tezJarResources.get(rsrcName).getResource().toString()
- + ", newPath=" + fStatus.getPath();
- LOG.warn(message);
- // throw new TezUncheckedException(message);
- }
- tezJarResources.put(rsrcName,
- LocalResource.newInstance(
- ConverterUtils.getYarnUrlFromPath(fStatus.getPath()),
- LocalResourceType.FILE,
- LocalResourceVisibility.PUBLIC,
- fStatus.getLen(),
- fStatus.getModificationTime()));
- }
- }
- if (tezJarResources.isEmpty()) {
- LOG.warn("No tez jars found in configured locations"
- + ". Ignoring for now. Errors may occur");
- }
- return tezJarResources;
- }
-
- /**
- * Verify or create the Staging area directory on the configured Filesystem
- * @param stagingArea Staging area directory path
- * @return
- * @throws IOException
- */
- public static FileSystem ensureStagingDirExists(Configuration conf,
- Path stagingArea)
- throws IOException {
- FileSystem fs = stagingArea.getFileSystem(conf);
- String realUser;
- String currentUser;
- UserGroupInformation ugi = UserGroupInformation.getLoginUser();
- realUser = ugi.getShortUserName();
- currentUser = UserGroupInformation.getCurrentUser().getShortUserName();
- if (fs.exists(stagingArea)) {
- FileStatus fsStatus = fs.getFileStatus(stagingArea);
- String owner = fsStatus.getOwner();
- if (!(owner.equals(currentUser) || owner.equals(realUser))) {
- throw new IOException("The ownership on the staging directory "
- + stagingArea + " is not as expected. " + "It is owned by " + owner
- + ". The directory must " + "be owned by the submitter "
- + currentUser + " or " + "by " + realUser);
- }
- if (!fsStatus.getPermission().equals(TEZ_AM_DIR_PERMISSION)) {
- LOG.info("Permissions on staging directory " + stagingArea + " are "
- + "incorrect: " + fsStatus.getPermission()
- + ". Fixing permissions " + "to correct value "
- + TEZ_AM_DIR_PERMISSION);
- fs.setPermission(stagingArea, TEZ_AM_DIR_PERMISSION);
- }
- } else {
- fs.mkdirs(stagingArea, new FsPermission(TEZ_AM_DIR_PERMISSION));
- }
- return fs;
- }
-
- /**
- * Create an ApplicationSubmissionContext to launch a Tez AM
- * @param conf
- * @param appId
- * @param dag
- * @param appStagingDir
- * @param ts
- * @param amQueueName
- * @param amName
- * @param amArgs
- * @param amEnv
- * @param amLocalResources
- * @param appConf
- * @return
- * @throws IOException
- * @throws YarnException
- */
- static ApplicationSubmissionContext createApplicationSubmissionContext(
- Configuration conf, ApplicationId appId, DAG dag, String amName,
- AMConfiguration amConfig,
- Map<String, LocalResource> tezJarResources)
- throws IOException, YarnException{
-
- FileSystem fs = TezClientUtils.ensureStagingDirExists(conf,
- amConfig.getStagingDir());
-
- // Setup resource requirements
- Resource capability = Records.newRecord(Resource.class);
- capability.setMemory(
- amConfig.getAMConf().getInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB,
- TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB_DEFAULT));
- capability.setVirtualCores(
- amConfig.getAMConf().getInt(TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES,
- TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES_DEFAULT));
- if (LOG.isDebugEnabled()) {
- LOG.debug("AppMaster capability = " + capability);
- }
-
- ByteBuffer securityTokens = null;
- // Setup security tokens
- if (amConfig.getCredentials() != null) {
- DataOutputBuffer dob = new DataOutputBuffer();
- amConfig.getCredentials().writeTokenStorageToStream(dob);
- securityTokens = ByteBuffer.wrap(dob.getData(), 0,
- dob.getLength());
- }
-
- // Setup the command to run the AM
- List<String> vargs = new ArrayList<String>(8);
- vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
-
- String amLogLevel = amConfig.getAMConf().get(
- TezConfiguration.TEZ_AM_LOG_LEVEL,
- TezConfiguration.TEZ_AM_LOG_LEVEL_DEFAULT);
- addLog4jSystemProperties(amLogLevel, vargs);
-
- vargs.add(amConfig.getAMConf().get(TezConfiguration.TEZ_AM_JAVA_OPTS,
- TezConfiguration.DEFAULT_TEZ_AM_JAVA_OPTS));
-
- vargs.add(TezConfiguration.TEZ_APPLICATION_MASTER_CLASS);
- vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR +
- File.separator + ApplicationConstants.STDOUT);
- vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR +
- File.separator + ApplicationConstants.STDERR);
-
-
- Vector<String> vargsFinal = new Vector<String>(8);
- // Final command
- StringBuilder mergedCommand = new StringBuilder();
- for (CharSequence str : vargs) {
- mergedCommand.append(str).append(" ");
- }
- vargsFinal.add(mergedCommand.toString());
-
- LOG.debug("Command to launch container for ApplicationMaster is : "
- + mergedCommand);
-
- // Setup the CLASSPATH in environment
- // i.e. add { Hadoop jars, job jar, CWD } to classpath.
- Map<String, String> environment = new HashMap<String, String>();
-
- boolean isMiniCluster =
- conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false);
- if (isMiniCluster) {
- Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
- System.getProperty("java.class.path"));
- }
-
- Apps.addToEnvironment(environment,
- Environment.CLASSPATH.name(),
- Environment.PWD.$());
-
- Apps.addToEnvironment(environment,
- Environment.CLASSPATH.name(),
- Environment.PWD.$() + File.separator + "*");
-
- // Add YARN/COMMON/HDFS jars to path
- if (!isMiniCluster) {
- for (String c : conf.getStrings(
- YarnConfiguration.YARN_APPLICATION_CLASSPATH,
- YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
- Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
- c.trim());
- }
- }
-
- if (amConfig.getEnv() != null) {
- for (Map.Entry<String, String> entry : amConfig.getEnv().entrySet()) {
- Apps.addToEnvironment(environment, entry.getKey(), entry.getValue());
- }
- }
-
- Map<String, LocalResource> localResources =
- new TreeMap<String, LocalResource>();
-
- if (amConfig.getLocalResources() != null) {
- localResources.putAll(amConfig.getLocalResources());
- }
- localResources.putAll(tezJarResources);
-
- // emit conf as PB file
- Configuration finalTezConf = createFinalTezConfForApp(amConfig.getAMConf());
- Path binaryConfPath = new Path(amConfig.getStagingDir(),
- TezConfiguration.TEZ_PB_BINARY_CONF_NAME + "." + appId.toString());
- FSDataOutputStream amConfPBOutBinaryStream = null;
- try {
- ConfigurationProto.Builder confProtoBuilder =
- ConfigurationProto.newBuilder();
- Iterator<Entry<String, String>> iter = finalTezConf.iterator();
- while (iter.hasNext()) {
- Entry<String, String> entry = iter.next();
- PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder();
- kvp.setKey(entry.getKey());
- kvp.setValue(entry.getValue());
- confProtoBuilder.addConfKeyValues(kvp);
- }
- //binary output
- amConfPBOutBinaryStream = FileSystem.create(fs, binaryConfPath,
- new FsPermission(TEZ_AM_FILE_PERMISSION));
- confProtoBuilder.build().writeTo(amConfPBOutBinaryStream);
- } finally {
- if(amConfPBOutBinaryStream != null){
- amConfPBOutBinaryStream.close();
- }
- }
-
- LocalResource binaryConfLRsrc =
- TezClientUtils.createLocalResource(fs,
- binaryConfPath, LocalResourceType.FILE,
- LocalResourceVisibility.APPLICATION);
- localResources.put(TezConfiguration.TEZ_PB_BINARY_CONF_NAME,
- binaryConfLRsrc);
-
- if(dag != null) {
- // Add tez jars to vertices too
- for (Vertex v : dag.getVertices()) {
- v.getTaskLocalResources().putAll(tezJarResources);
- v.getTaskLocalResources().put(TezConfiguration.TEZ_PB_BINARY_CONF_NAME,
- binaryConfLRsrc);
- }
-
- // emit protobuf DAG file style
- Path binaryPath = new Path(amConfig.getStagingDir(),
- TezConfiguration.TEZ_PB_PLAN_BINARY_NAME + "." + appId.toString());
- amConfig.getAMConf().set(TezConfiguration.TEZ_AM_PLAN_REMOTE_PATH,
- binaryPath.toUri().toString());
-
- DAGPlan dagPB = dag.createDag(null);
-
- FSDataOutputStream dagPBOutBinaryStream = null;
-
- try {
- //binary output
- dagPBOutBinaryStream = FileSystem.create(fs, binaryPath,
- new FsPermission(TEZ_AM_FILE_PERMISSION));
- dagPB.writeTo(dagPBOutBinaryStream);
- } finally {
- if(dagPBOutBinaryStream != null){
- dagPBOutBinaryStream.close();
- }
- }
-
- localResources.put(TezConfiguration.TEZ_PB_PLAN_BINARY_NAME,
- TezClientUtils.createLocalResource(fs,
- binaryPath, LocalResourceType.FILE,
- LocalResourceVisibility.APPLICATION));
-
- if (Level.DEBUG.isGreaterOrEqual(Level.toLevel(amLogLevel))) {
- Path textPath = localizeDagPlanAsText(dagPB, fs,
- amConfig.getStagingDir(), appId);
- localResources.put(TezConfiguration.TEZ_PB_PLAN_TEXT_NAME,
- TezClientUtils.createLocalResource(fs,
- textPath, LocalResourceType.FILE,
- LocalResourceVisibility.APPLICATION));
- }
- } else {
- Apps.addToEnvironment(environment,
- TezConstants.TEZ_AM_IS_SESSION_ENV, "set");
- }
-
- Map<ApplicationAccessType, String> acls
- = new HashMap<ApplicationAccessType, String>();
-
- // Setup ContainerLaunchContext for AM container
- ContainerLaunchContext amContainer =
- ContainerLaunchContext.newInstance(localResources, environment,
- vargsFinal, null, securityTokens, acls);
-
- // Set up the ApplicationSubmissionContext
- ApplicationSubmissionContext appContext = Records
- .newRecord(ApplicationSubmissionContext.class);
-
- appContext.setApplicationType(TezConfiguration.TEZ_APPLICATION_TYPE);
- appContext.setApplicationId(appId);
- appContext.setResource(capability);
- appContext.setQueue(amConfig.getQueueName());
- appContext.setApplicationName(amName);
- appContext.setCancelTokensWhenComplete(amConfig.getAMConf().getBoolean(
- TezConfiguration.TEZ_AM_CANCEL_DELEGATION_TOKEN,
- TezConfiguration.TEZ_AM_CANCEL_DELEGATION_TOKEN_DEFAULT));
- appContext.setAMContainerSpec(amContainer);
-
- return appContext;
-
- }
-
- @VisibleForTesting
- static void addLog4jSystemProperties(String logLevel,
- List<String> vargs) {
- vargs.add("-Dlog4j.configuration="
- + TezConfiguration.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
- vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
- + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
- vargs.add("-D" + TezConfiguration.TEZ_ROOT_LOGGER_NAME + "=" + logLevel
- + "," + TezConfiguration.TEZ_CONTAINER_LOGGER_NAME);
- }
-
- static Configuration createFinalTezConfForApp(TezConfiguration amConf) {
- Configuration conf = new Configuration(false);
- conf.setQuietMode(true);
-
- assert amConf != null;
- Iterator<Entry<String, String>> iter = amConf.iterator();
- while (iter.hasNext()) {
- Entry<String, String> entry = iter.next();
- // Copy all tez config parameters.
- if (entry.getKey().startsWith(TezConfiguration.TEZ_PREFIX)) {
- conf.set(entry.getKey(), entry.getValue());
- if (LOG.isDebugEnabled()) {
- LOG.debug("Adding tez dag am parameter: " + entry.getKey()
- + ", with value: " + entry.getValue());
- }
- }
- }
- return conf;
- }
-
- /**
- * Helper function to create a YARN LocalResource
- * @param fs FileSystem object
- * @param p Path of resource to localize
- * @param type LocalResource Type
- * @return
- * @throws IOException
- */
- static LocalResource createLocalResource(FileSystem fs, Path p,
- LocalResourceType type,
- LocalResourceVisibility visibility) throws IOException {
- LocalResource rsrc = Records.newRecord(LocalResource.class);
- FileStatus rsrcStat = fs.getFileStatus(p);
- rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs.resolvePath(rsrcStat
- .getPath())));
- rsrc.setSize(rsrcStat.getLen());
- rsrc.setTimestamp(rsrcStat.getModificationTime());
- rsrc.setType(type);
- rsrc.setVisibility(visibility);
- return rsrc;
- }
-
- private static Path localizeDagPlanAsText(DAGPlan dagPB, FileSystem fs,
- Path appStagingDir, ApplicationId appId) throws IOException {
- Path textPath = new Path(appStagingDir,
- TezConfiguration.TEZ_PB_PLAN_TEXT_NAME + "." + appId.toString());
- FSDataOutputStream dagPBOutTextStream = null;
- try {
- dagPBOutTextStream = FileSystem.create(fs, textPath, new FsPermission(
- TEZ_AM_FILE_PERMISSION));
- String dagPBStr = dagPB.toString();
- int dagPBStrLen = dagPBStr.length();
- if (dagPBStrLen <= UTF8_CHUNK_SIZE) {
- dagPBOutTextStream.writeUTF(dagPBStr);
- } else {
- int startIndex = 0;
- while (startIndex < dagPBStrLen) {
- int endIndex = startIndex + UTF8_CHUNK_SIZE;
- if (endIndex > dagPBStrLen) {
- endIndex = dagPBStrLen;
- }
- dagPBOutTextStream.writeUTF(dagPBStr.substring(startIndex, endIndex));
- startIndex += UTF8_CHUNK_SIZE;
- }
- }
- } finally {
- if (dagPBOutTextStream != null) {
- dagPBOutTextStream.close();
- }
- }
- return textPath;
- }
-
- static DAGClientAMProtocolBlockingPB getAMProxy(YarnClient yarnClient,
- Configuration conf,
- ApplicationId applicationId) throws TezException, IOException {
- ApplicationReport appReport;
- try {
- appReport = yarnClient.getApplicationReport(
- applicationId);
-
- if(appReport == null) {
- throw new TezUncheckedException("Could not retrieve application report"
- + " from YARN, applicationId=" + applicationId);
- }
- YarnApplicationState appState = appReport.getYarnApplicationState();
- if(appState != YarnApplicationState.RUNNING) {
- if (appState == YarnApplicationState.FINISHED
- || appState == YarnApplicationState.KILLED
- || appState == YarnApplicationState.FAILED) {
- throw new TezUncheckedException("Application not running"
- + ", applicationId=" + applicationId
- + ", yarnApplicationState=" + appReport.getYarnApplicationState()
- + ", finalApplicationStatus="
- + appReport.getFinalApplicationStatus()
- + ", trackingUrl=" + appReport.getTrackingUrl());
- }
- return null;
- }
- } catch (YarnException e) {
- throw new TezException(e);
- }
- return getAMProxy(conf, appReport.getHost(), appReport.getRpcPort());
- }
-
- static DAGClientAMProtocolBlockingPB getAMProxy(Configuration conf,
- String amHost, int amRpcPort) throws IOException {
- InetSocketAddress addr = new InetSocketAddress(amHost,
- amRpcPort);
-
- RPC.setProtocolEngine(conf, DAGClientAMProtocolBlockingPB.class,
- ProtobufRpcEngine.class);
- DAGClientAMProtocolBlockingPB proxy =
- (DAGClientAMProtocolBlockingPB) RPC.getProxy(
- DAGClientAMProtocolBlockingPB.class, 0, addr, conf);
- return proxy;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-dag-api/src/main/java/org/apache/tez/client/TezSession.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/client/TezSession.java b/tez-dag-api/src/main/java/org/apache/tez/client/TezSession.java
deleted file mode 100644
index acf523d..0000000
--- a/tez-dag-api/src/main/java/org/apache/tez/client/TezSession.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.client;
-
-import java.io.IOException;
-import java.util.Map;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.api.Vertex;
-import org.apache.tez.dag.api.client.DAGClient;
-import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolBlockingPB;
-import org.apache.tez.dag.api.client.rpc.DAGClientRPCImpl;
-import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.ShutdownSessionRequestProto;
-import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.SubmitDAGRequestProto;
-import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.protobuf.ServiceException;
-
-public class TezSession {
-
- private static final Log LOG = LogFactory.getLog(TezSession.class);
-
- private final String sessionName;
- private ApplicationId applicationId;
- private LocalResource tezConfPBLRsrc = null;
- private final TezSessionConfiguration sessionConfig;
- private YarnClient yarnClient;
- private Map<String, LocalResource> tezJarResources;
- private boolean sessionStarted = false;
-
- public TezSession(String sessionName,
- ApplicationId applicationId,
- TezSessionConfiguration sessionConfig) {
- this.sessionName = sessionName;
- this.sessionConfig = sessionConfig;
- this.applicationId = applicationId;
- }
-
- public TezSession(String sessionName,
- TezSessionConfiguration sessionConfig) {
- this(sessionName, null, sessionConfig);
- }
-
- public synchronized void start() throws TezException, IOException {
- yarnClient = YarnClient.createYarnClient();
- yarnClient.init(sessionConfig.getYarnConfiguration());
- yarnClient.start();
-
- tezJarResources = TezClientUtils.setupTezJarsLocalResources(
- sessionConfig.getTezConfiguration());
-
- try {
- if (applicationId == null) {
- applicationId = yarnClient.createApplication().
- getNewApplicationResponse().getApplicationId();
- }
-
- ApplicationSubmissionContext appContext =
- TezClientUtils.createApplicationSubmissionContext(
- sessionConfig.getTezConfiguration(), applicationId,
- null, sessionName, sessionConfig.getAMConfiguration(),
- tezJarResources);
- tezConfPBLRsrc = appContext.getAMContainerSpec().getLocalResources().get(
- TezConfiguration.TEZ_PB_BINARY_CONF_NAME);
- yarnClient.submitApplication(appContext);
- } catch (YarnException e) {
- throw new TezException(e);
- }
- sessionStarted = true;
- }
-
- public synchronized DAGClient submitDAG(DAG dag)
- throws TezException, IOException {
- if (!sessionStarted) {
- throw new TezUncheckedException("Session not started");
- }
-
- String dagId = null;
- LOG.info("Submitting dag to TezSession"
- + ", sessionName=" + sessionName
- + ", applicationId=" + applicationId);
- // Add tez jars to vertices too
- for (Vertex v : dag.getVertices()) {
- v.getTaskLocalResources().putAll(tezJarResources);
- if (null != tezConfPBLRsrc) {
- v.getTaskLocalResources().put(TezConfiguration.TEZ_PB_BINARY_CONF_NAME,
- tezConfPBLRsrc);
- }
- }
- DAGPlan dagPlan = dag.createDag(sessionConfig.getTezConfiguration());
- SubmitDAGRequestProto requestProto =
- SubmitDAGRequestProto.newBuilder().setDAGPlan(dagPlan).build();
-
- DAGClientAMProtocolBlockingPB proxy;
- while (true) {
- proxy = TezClientUtils.getAMProxy(yarnClient,
- sessionConfig.getYarnConfiguration(), applicationId);
- if (proxy != null) {
- break;
- }
- try {
- Thread.sleep(100l);
- } catch (InterruptedException e) {
- // Ignore
- }
- }
-
- try {
- dagId = proxy.submitDAG(null, requestProto).getDagId();
- } catch (ServiceException e) {
- throw new TezException(e);
- }
- LOG.info("Submitted dag to TezSession"
- + ", sessionName=" + sessionName
- + ", applicationId=" + applicationId
- + ", dagId=" + dagId);
- return new DAGClientRPCImpl(applicationId, dagId,
- sessionConfig.getTezConfiguration());
- }
-
- public synchronized void stop() throws TezException, IOException {
- LOG.info("Shutting down Tez Session"
- + ", sessionName=" + sessionName
- + ", applicationId=" + applicationId);
- DAGClientAMProtocolBlockingPB proxy = TezClientUtils.getAMProxy(yarnClient,
- sessionConfig.getYarnConfiguration(), applicationId);
- if (proxy != null) {
- try {
- ShutdownSessionRequestProto request =
- ShutdownSessionRequestProto.newBuilder().build();
- proxy.shutdownSession(null, request);
- return;
- } catch (ServiceException e) {
- LOG.info("Failed to shutdown Tez Session via proxy", e);
- }
- }
- LOG.info("Could not connect to AM, killing session via YARN"
- + ", sessionName=" + sessionName
- + ", applicationId=" + applicationId);
- try {
- yarnClient.killApplication(applicationId);
- } catch (YarnException e) {
- throw new TezException(e);
- }
- }
-
- public String getSessionName() {
- return sessionName;
- }
-
- @Private
- @VisibleForTesting
- public synchronized ApplicationId getApplicationId() {
- return applicationId;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d316f723/tez-dag-api/src/main/java/org/apache/tez/client/TezSessionConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/client/TezSessionConfiguration.java b/tez-dag-api/src/main/java/org/apache/tez/client/TezSessionConfiguration.java
deleted file mode 100644
index 61ca60b..0000000
--- a/tez-dag-api/src/main/java/org/apache/tez/client/TezSessionConfiguration.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.client;
-
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.tez.dag.api.TezConfiguration;
-
-public class TezSessionConfiguration {
-
- private final AMConfiguration amConfiguration;
- private final YarnConfiguration yarnConfig;
- private final TezConfiguration tezConfig;
-
- public TezSessionConfiguration(AMConfiguration amConfiguration,
- TezConfiguration tezConfig) {
- this.amConfiguration = amConfiguration;
- this.tezConfig = tezConfig;
- this.yarnConfig = new YarnConfiguration(tezConfig);
- }
-
- TezSessionConfiguration(AMConfiguration amConfiguration,
- TezConfiguration tezConfig,
- YarnConfiguration yarnConf) {
- this.amConfiguration = amConfiguration;
- this.tezConfig = tezConfig;
- this.yarnConfig = yarnConf;
- }
-
- public AMConfiguration getAMConfiguration() {
- return amConfiguration;
- }
-
- public YarnConfiguration getYarnConfiguration() {
- return yarnConfig;
- }
-
- public TezConfiguration getTezConfiguration() {
- return tezConfig;
- }
-
-}