You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2014/10/27 18:38:31 UTC
svn commit: r1634612 - in /hive/branches/spark/spark-client: ./
src/main/java/org/apache/hive/spark/client/
src/main/java/org/apache/hive/spark/client/metrics/
Author: xuefu
Date: Mon Oct 27 17:38:31 2014
New Revision: 1634612
URL: http://svn.apache.org/r1634612
Log:
HIVE-8599: Add InterfaceAudience annotations to spark-client [Spark Branch] (Marcelo via Xuefu)
Modified:
hive/branches/spark/spark-client/pom.xml
hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/Job.java
hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/JobContext.java
hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/JobHandle.java
hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/MetricsCollection.java
hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java
hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClient.java
hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientFactory.java
hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/DataReadMethod.java
hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/InputMetrics.java
hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/Metrics.java
hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/ShuffleReadMetrics.java
hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/ShuffleWriteMetrics.java
Modified: hive/branches/spark/spark-client/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/pom.xml?rev=1634612&r1=1634611&r2=1634612&view=diff
==============================================================================
--- hive/branches/spark/spark-client/pom.xml (original)
+++ hive/branches/spark/spark-client/pom.xml Mon Oct 27 17:38:31 2014
@@ -52,6 +52,11 @@
<artifactId>guava</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
</dependency>
Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/Job.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/Job.java?rev=1634612&r1=1634611&r2=1634612&view=diff
==============================================================================
--- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/Job.java (original)
+++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/Job.java Mon Oct 27 17:38:31 2014
@@ -19,10 +19,13 @@ package org.apache.hive.spark.client;
import java.io.Serializable;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+
/**
* Interface for a Spark remote job.
*/
-interface Job<T extends Serializable> extends Serializable {
+@InterfaceAudience.Private
+public interface Job<T extends Serializable> extends Serializable {
T call(JobContext jc) throws Exception;
Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/JobContext.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/JobContext.java?rev=1634612&r1=1634611&r2=1634612&view=diff
==============================================================================
--- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/JobContext.java (original)
+++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/JobContext.java Mon Oct 27 17:38:31 2014
@@ -20,12 +20,15 @@ package org.apache.hive.spark.client;
import org.apache.spark.api.java.JavaFutureAction;
import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+
/**
* Holds runtime information about the job execution context.
*
* An instance of this class is kept on the node hosting a remote Spark context and is made
* available to jobs being executed via RemoteSparkContext#submit().
*/
+@InterfaceAudience.Private
public interface JobContext {
/** The shared SparkContext instance. */
Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/JobHandle.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/JobHandle.java?rev=1634612&r1=1634611&r2=1634612&view=diff
==============================================================================
--- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/JobHandle.java (original)
+++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/JobHandle.java Mon Oct 27 17:38:31 2014
@@ -20,10 +20,13 @@ package org.apache.hive.spark.client;
import java.io.Serializable;
import java.util.concurrent.Future;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+
/**
* A handle to a submitted job. Allows for monitoring and controlling of the running remote job.
*/
-interface JobHandle<T extends Serializable> extends Future<T> {
+@InterfaceAudience.Private
+public interface JobHandle<T extends Serializable> extends Future<T> {
/**
* The client job ID. This is unrelated to any Spark jobs that might be triggered by the
Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/MetricsCollection.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/MetricsCollection.java?rev=1634612&r1=1634611&r2=1634612&view=diff
==============================================================================
--- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/MetricsCollection.java (original)
+++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/MetricsCollection.java Mon Oct 27 17:38:31 2014
@@ -33,6 +33,7 @@ import com.google.common.collect.Collect
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hive.spark.client.metrics.DataReadMethod;
import org.apache.hive.spark.client.metrics.InputMetrics;
import org.apache.hive.spark.client.metrics.Metrics;
@@ -51,6 +52,7 @@ import org.apache.hive.spark.client.metr
* Only successful, non-speculative tasks are considered. Metrics are updated as tasks finish,
* so snapshots can be retrieved before the whole job completes.
*/
+@InterfaceAudience.Private
public class MetricsCollection {
private final List<TaskInfo> taskMetrics = Lists.newArrayList();
Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java?rev=1634612&r1=1634611&r2=1634612&view=diff
==============================================================================
--- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java (original)
+++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java Mon Oct 27 17:38:31 2014
@@ -46,11 +46,13 @@ import org.apache.spark.api.java.JavaSpa
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hive.spark.client.metrics.Metrics;
/**
* Driver code for the Spark client library.
*/
+@InterfaceAudience.Private
public class RemoteDriver {
private final static Logger LOG = LoggerFactory.getLogger(RemoteDriver.class);
Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClient.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClient.java?rev=1634612&r1=1634611&r2=1634612&view=diff
==============================================================================
--- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClient.java (original)
+++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClient.java Mon Oct 27 17:38:31 2014
@@ -21,9 +21,12 @@ import java.net.URL;
import java.io.Serializable;
import java.util.concurrent.Future;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+
/**
* Defines the API for the Spark remote client.
*/
+@InterfaceAudience.Private
public interface SparkClient {
/**
Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientFactory.java?rev=1634612&r1=1634611&r2=1634612&view=diff
==============================================================================
--- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientFactory.java (original)
+++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientFactory.java Mon Oct 27 17:38:31 2014
@@ -24,9 +24,12 @@ import akka.actor.ActorSystem;
import com.google.common.collect.Maps;
import org.apache.spark.SparkException;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+
/**
* Factory for SparkClient instances.
*/
+@InterfaceAudience.Private
public final class SparkClientFactory {
static ActorSystem actorSystem = null;
Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/DataReadMethod.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/DataReadMethod.java?rev=1634612&r1=1634611&r2=1634612&view=diff
==============================================================================
--- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/DataReadMethod.java (original)
+++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/DataReadMethod.java Mon Oct 27 17:38:31 2014
@@ -17,10 +17,13 @@
package org.apache.hive.spark.client.metrics;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+
/**
* Method by which input data was read. Network means that the data was read over the network
* from a remote block manager (which may have stored the data on-disk or in-memory).
*/
+@InterfaceAudience.Private
public enum DataReadMethod {
Memory, Disk, Hadoop, Network, Multiple
}
Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/InputMetrics.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/InputMetrics.java?rev=1634612&r1=1634611&r2=1634612&view=diff
==============================================================================
--- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/InputMetrics.java (original)
+++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/InputMetrics.java Mon Oct 27 17:38:31 2014
@@ -21,9 +21,12 @@ import java.io.Serializable;
import org.apache.spark.executor.TaskMetrics;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+
/**
* Metrics pertaining to reading input data.
*/
+@InterfaceAudience.Private
public class InputMetrics implements Serializable {
public final DataReadMethod readMethod;
Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/Metrics.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/Metrics.java?rev=1634612&r1=1634611&r2=1634612&view=diff
==============================================================================
--- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/Metrics.java (original)
+++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/Metrics.java Mon Oct 27 17:38:31 2014
@@ -22,12 +22,15 @@ import java.io.Serializable;
import com.google.common.base.Optional;
import org.apache.spark.executor.TaskMetrics;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+
/**
* Metrics tracked during the execution of a job.
*
* Depending on how the metrics object is obtained (by calling methods in the `MetricsCollection`
* class), metrics will refer to one or more tasks.
*/
+@InterfaceAudience.Private
public class Metrics implements Serializable {
/** Time taken on the executor to deserialize tasks. */
Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/ShuffleReadMetrics.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/ShuffleReadMetrics.java?rev=1634612&r1=1634611&r2=1634612&view=diff
==============================================================================
--- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/ShuffleReadMetrics.java (original)
+++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/ShuffleReadMetrics.java Mon Oct 27 17:38:31 2014
@@ -21,9 +21,12 @@ import java.io.Serializable;
import org.apache.spark.executor.TaskMetrics;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+
/**
* Metrics pertaining to reading shuffle data.
*/
+@InterfaceAudience.Private
public class ShuffleReadMetrics implements Serializable {
/** Number of remote blocks fetched in shuffles by tasks. */
Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/ShuffleWriteMetrics.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/ShuffleWriteMetrics.java?rev=1634612&r1=1634611&r2=1634612&view=diff
==============================================================================
--- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/ShuffleWriteMetrics.java (original)
+++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/metrics/ShuffleWriteMetrics.java Mon Oct 27 17:38:31 2014
@@ -21,9 +21,12 @@ import java.io.Serializable;
import org.apache.spark.executor.TaskMetrics;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+
/**
* Metrics pertaining to writing shuffle data.
*/
+@InterfaceAudience.Private
public class ShuffleWriteMetrics implements Serializable {
/** Number of bytes written for the shuffle by tasks. */